From b954ff3268e74f50d28ada5c652089a9fc526866 Mon Sep 17 00:00:00 2001 From: Thomas Gardner Date: Wed, 3 Feb 2016 18:51:32 +1000 Subject: [PATCH 01/21] repo: add `ipfs repo stat` command License: MIT Signed-off-by: Thomas Gardner --- core/commands/repo.go | 69 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index 3b634e6311c..7af24f8bf51 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -4,12 +4,20 @@ import ( "bytes" "fmt" "io" + "strings" cmds "github.com/ipfs/go-ipfs/commands" corerepo "github.com/ipfs/go-ipfs/core/corerepo" + fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) +type RepoStat struct { + repoPath string + repoSize uint64 // size in bytes + numBlocks uint64 +} + var RepoCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Manipulate the IPFS repo.", @@ -19,7 +27,8 @@ var RepoCmd = &cmds.Command{ }, Subcommands: map[string]*cmds.Command{ - "gc": repoGcCmd, + "gc": repoGcCmd, + "stat": repoStatCmd, }, } @@ -95,3 +104,61 @@ order to reclaim hard disk space. }, }, } + +var repoStatCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Print status of the local repo.", + ShortDescription: ``, + }, + Run: func(req cmds.Request, res cmds.Response) { + ctx := req.Context() + n, err := req.InvocContext().GetNode() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + usage, err := n.Repo.GetStorageUsage() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + allKeys, err := n.Blockstore.AllKeysChan(ctx) + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + count := uint64(0) + for range allKeys { + count++ + } + + path, err := fsrepo.BestKnownPath() + if err != nil { + res.SetError(err, cmds.ErrNormal) + return + } + + out := &RepoStat{ + repoPath: path, + repoSize: usage, + numBlocks: count, + } + res.SetOutput(out) + }, + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + stat, ok := res.Output().(*RepoStat) + if !ok { + return nil, u.ErrCast() + } + + out := fmt.Sprintf("Path: %s\nSize: %d bytes\n"+ + "Blocks: %d\n", + stat.repoPath, stat.repoSize, stat.numBlocks) + return strings.NewReader(out), nil + }, + }, +} From 8aa1cc2f6d3ea66a273a245ab91ccaf6a4375153 Mon Sep 17 00:00:00 2001 From: Thomas Gardner Date: Fri, 5 Feb 2016 11:22:49 +1000 Subject: [PATCH 02/21] `repo stat`: add Type License: MIT Signed-off-by: Thomas Gardner --- core/commands/repo.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index 7af24f8bf51..445d3a9c659 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -141,13 +141,13 @@ var repoStatCmd = &cmds.Command{ return } - out := &RepoStat{ + res.SetOutput(&RepoStat{ repoPath: path, repoSize: usage, numBlocks: count, - } - res.SetOutput(out) + }) }, + Type: RepoStat{}, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { stat, ok := res.Output().(*RepoStat) @@ -155,9 +155,10 @@ var repoStatCmd = &cmds.Command{ return nil, u.ErrCast() } - out := fmt.Sprintf("Path: %s\nSize: %d bytes\n"+ - "Blocks: %d\n", + out := fmt.Sprintf( + "Path: %s\nSize: %d bytes\nBlocks: %d\n", stat.repoPath, stat.repoSize, stat.numBlocks) + return strings.NewReader(out), nil }, }, From ae7d672f0d34b2e97547affc19f11ba6f441a0a2 Mon Sep 17 00:00:00 2001 From: Thomas Gardner Date: Wed, 10 Feb 2016 20:07:31 +1000 Subject: [PATCH 03/21] Humanize byte size License: MIT Signed-off-by: Thomas Gardner --- core/commands/repo.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/commands/repo.go b/core/commands/repo.go index 445d3a9c659..7054ae9821b 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -6,6 +6,7 @@ import ( "io" "strings" + humanize "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize" cmds "github.com/ipfs/go-ipfs/commands" corerepo "github.com/ipfs/go-ipfs/core/corerepo" fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" @@ -156,8 +157,8 @@ var repoStatCmd = &cmds.Command{ } out := fmt.Sprintf( - "Path: %s\nSize: %d bytes\nBlocks: %d\n", - stat.repoPath, stat.repoSize, stat.numBlocks) + "Path: %s\nSize: %s\nBlocks: %d\n", + stat.repoPath, humanize.Bytes(stat.repoSize), stat.numBlocks) return strings.NewReader(out), nil }, From 5726f29a553ab363fdfdcecfb9213eec5159058a Mon Sep 17 00:00:00 2001 From: David Dias Date: Wed, 2 Mar 2016 08:48:51 +0000 Subject: [PATCH 04/21] repo-stat License: MIT Signed-off-by: David Dias --- core/commands/repo.go | 74 ++++++++++++++++--------------------- core/corerepo/stat.go | 43 +++++++++++++++++++++ test/sharness/t0080-repo.sh | 25 +++++++++++++ 3 files changed, 99 insertions(+), 43 deletions(-) create mode 100644 core/corerepo/stat.go diff --git a/core/commands/repo.go b/core/commands/repo.go index 7054ae9821b..e4ac8807d64 100644 --- a/core/commands/repo.go +++ b/core/commands/repo.go @@ -3,22 +3,12 @@ package commands import ( "bytes" "fmt" - "io" - "strings" - - humanize "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize" cmds "github.com/ipfs/go-ipfs/commands" corerepo "github.com/ipfs/go-ipfs/core/corerepo" - fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" + "io" ) -type RepoStat struct { - repoPath string - repoSize uint64 // size in bytes - numBlocks uint64 -} - var RepoCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Manipulate the IPFS repo.", @@ -108,59 +98,57 @@ order to reclaim hard disk space. var repoStatCmd = &cmds.Command{ Helptext: cmds.HelpText{ - Tagline: "Print status of the local repo.", - ShortDescription: ``, + Tagline: "Get stats for the currently used repo.", + ShortDescription: ` +'ipfs repo stat' is a plumbing command that will scan the local +set of stored objects and print repo statistics. It outputs to stdout: +NumObjects int number of objects in the local repo +RepoSize int size in bytes that the repo is currently taking +RepoPath string the path to the repo being currently used +`, }, Run: func(req cmds.Request, res cmds.Response) { - ctx := req.Context() n, err := req.InvocContext().GetNode() if err != nil { res.SetError(err, cmds.ErrNormal) return } - usage, err := n.Repo.GetStorageUsage() - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - - allKeys, err := n.Blockstore.AllKeysChan(ctx) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - - count := uint64(0) - for range allKeys { - count++ - } - - path, err := fsrepo.BestKnownPath() + stat, err := corerepo.RepoStat(n, req.Context()) if err != nil { res.SetError(err, cmds.ErrNormal) return } - res.SetOutput(&RepoStat{ - repoPath: path, - repoSize: usage, - numBlocks: count, - }) + res.SetOutput(stat) }, - Type: RepoStat{}, + Options: []cmds.Option{ + cmds.BoolOption("human", "Output RepoSize in MiB."), + }, + Type: corerepo.Stat{}, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - stat, ok := res.Output().(*RepoStat) + stat, ok := res.Output().(*corerepo.Stat) if !ok { return nil, u.ErrCast() } - out := fmt.Sprintf( - "Path: %s\nSize: %s\nBlocks: %d\n", - stat.repoPath, humanize.Bytes(stat.repoSize), stat.numBlocks) + human, _, err := res.Request().Option("human").Bool() + if err != nil { + return nil, err + } + + buf := new(bytes.Buffer) + fmt.Fprintf(buf, "NumObjects \t %d\n", stat.NumObjects) + sizeInMiB := stat.RepoSize / (1024 * 1024) + if human && sizeInMiB > 0 { + fmt.Fprintf(buf, "RepoSize (MiB) \t %d\n", sizeInMiB) + } else { + fmt.Fprintf(buf, "RepoSize \t %d\n", stat.RepoSize) + } + fmt.Fprintf(buf, "RepoPath \t %s\n", stat.RepoPath) - return strings.NewReader(out), nil + return buf, nil }, }, } diff --git a/core/corerepo/stat.go b/core/corerepo/stat.go new file mode 100644 index 00000000000..65abf2efd42 --- /dev/null +++ b/core/corerepo/stat.go @@ -0,0 +1,43 @@ +package corerepo + +import ( + "github.com/ipfs/go-ipfs/core" + fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo" + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" +) + +type Stat struct { + NumObjects uint64 + RepoSize uint64 // size in bytes + RepoPath string +} + +func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) { + r := n.Repo + + usage, err := r.GetStorageUsage() + if err != nil { + return nil, err + } + + allKeys, err := n.Blockstore.AllKeysChan(ctx) + if err != nil { + return nil, err + } + + count := uint64(0) + for range allKeys { + count++ + } + + path, err := fsrepo.BestKnownPath() + if err != nil { + return nil, err + } + + return &Stat{ + NumObjects: count, + RepoSize: usage, + RepoPath: path, + }, nil +} diff --git a/test/sharness/t0080-repo.sh b/test/sharness/t0080-repo.sh index 01ef79b0ab3..cbe4e8dcf2d 100755 --- a/test/sharness/t0080-repo.sh +++ b/test/sharness/t0080-repo.sh @@ -219,6 +219,31 @@ test_expect_success "'ipfs refs --unique --recursive (bigger)'" ' test_sort_cmp expected actual || test_fsh cat refs_output ' +get_field_num() { + field=$1 + file=$2 + num=$(grep "$field" "$file" | awk '{ print $2 }') + echo $num +} + +test_expect_success "'ipfs repo stat' succeeds" ' + ipfs repo stat > repo-stats +' +test_expect_success "repo stats came out correct" ' + grep "RepoPath" repo-stats && + grep "RepoSize" repo-stats && + grep "NumObjects" repo-stats +' + +test_expect_success "'ipfs repo stat' after adding a file" ' + ipfs add repo-stats && + ipfs repo stat > repo-stats-2 +' + +test_expect_success "repo stats are updated correctly" ' + test $(get_field_num "RepoSize" repo-stats-2) -ge $(get_field_num "RepoSize" repo-stats) +' + test_kill_ipfs_daemon test_done From ea870a15a76457c2f567308989b8636d341d08cd Mon Sep 17 00:00:00 2001 From: Christian Couder Date: Mon, 7 Mar 2016 10:35:52 +0100 Subject: [PATCH 05/21] t0300: improve docker_build debug message License: MIT Signed-off-by: Christian Couder --- test/sharness/t0300-docker-image.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0300-docker-image.sh b/test/sharness/t0300-docker-image.sh index 478b2434450..ec964cd0d7e 100755 --- a/test/sharness/t0300-docker-image.sh +++ b/test/sharness/t0300-docker-image.sh @@ -33,7 +33,10 @@ TEST_TESTS_DIR=$(dirname "$TEST_SCRIPTS_DIR") APP_ROOT_DIR=$(dirname "$TEST_TESTS_DIR") test_expect_success "docker image build succeeds" ' - docker_build "$TEST_TESTS_DIR/Dockerfile" "$APP_ROOT_DIR" >actual + docker_build "$TEST_TESTS_DIR/Dockerfile" "$APP_ROOT_DIR" >actual || + test_fsh echo "TEST_TESTS_DIR: $TEST_TESTS_DIR" || + test_fsh echo "APP_ROOT_DIR : $APP_ROOT_DIR" || + test_fsh cat actual ' test_expect_success "docker image build output looks good" ' From 129656ea171b3d6420161a25aea69873c5ab15b2 Mon Sep 17 00:00:00 2001 From: hutenosa Date: Mon, 7 Mar 2016 22:51:23 +0100 Subject: [PATCH 06/21] build: include commit hash & version IPFS version is taken from git describe output. Done for Makefile and Dockerfile License: MIT Signed-off-by: hutenosa --- Dockerfile | 10 ++++-- Makefile | 7 ++-- bin/genversion | 78 ++++++++++++++++++++++++++++++++++++++++ core/commands/version.go | 2 +- repo/config/version.go | 6 ++-- test/Dockerfile | 9 +++-- 6 files changed, 102 insertions(+), 10 deletions(-) create mode 100755 bin/genversion diff --git a/Dockerfile b/Dockerfile index b641c4b463d..01388a7282d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,9 @@ ENV GOPATH /go ENV PATH /go/bin:$PATH ENV SRC_PATH /go/src/github.com/ipfs/go-ipfs +# Change to ARG once travis gets updated docker (1.9+) +ENV IPFS_VERSION=0.4.0-rc2 + # Get the go-ipfs sourcecode COPY . $SRC_PATH @@ -53,10 +56,11 @@ RUN apk add --update musl go=$GO_VERSION git bash wget ca-certificates \ # This saves us quite a bit of image size. && ref="$(cat .git/HEAD | cut -d' ' -f2)" \ && commit="$(cat .git/$ref | head -c 7)" \ - && echo "ldflags=-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$commit" \ - # Build and install IPFS and entrypoint script + && ldflags="-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$commit \ + -X github.com/ipfs/go-ipfs/repo/config.CurrentVersionNumber=$IPFS_VERSION" \ + && echo "ldflags: $ldflags" \ && cd $SRC_PATH/cmd/ipfs \ - && go build -ldflags "-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$commit" \ + && go build -ldflags $ldflags \ && cp ipfs /usr/local/bin/ipfs \ && cp $SRC_PATH/bin/container_daemon /usr/local/bin/start_ipfs \ && chmod 755 /usr/local/bin/start_ipfs \ diff --git a/Makefile b/Makefile index 79509bd8f84..04d69d84a7b 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,11 @@ else go_test=go test endif -COMMIT := $(shell git rev-parse --short HEAD) -ldflags = "-X "github.com/ipfs/go-ipfs/repo/config".CurrentCommit=$(COMMIT)" +VERSION := $(shell bin/genversion --version) +COMMIT := $(shell bin/genversion --commit) +ldflags = "-X "github.com/ipfs/go-ipfs/repo/config".CurrentVersionNumber=$(VERSION) \ +-X "github.com/ipfs/go-ipfs/repo/config".CurrentCommit=$(COMMIT)" + MAKEFLAGS += --no-print-directory diff --git a/bin/genversion b/bin/genversion new file mode 100755 index 00000000000..030aa6d9847 --- /dev/null +++ b/bin/genversion @@ -0,0 +1,78 @@ +#!/bin/bash + +# +# Calls git describe and extracts SemVer-compatible version and meta info +# Assumes git tags are properly formatted: vX.Y.Z or vX.Y.Z-some-tag +# +# Examples: +# +# $ genversion --version +# 0.4.0-rc2 +# +# or: +# +# $ genversion --commit +# c148.91c6f0f.dirty +# +# (148 commits ahead of the version 0.4.0-rc2, dirty workdir with HEAD at 91c6f0f) + + +PRINT_VERSION=0 +PRINT_COMMIT=0 + +test $# -ge 1 || PRINT_VERSION=1 + +while test $# -gt 0; do + case "$1" in + "--version") + PRINT_VERSION=1 + ;; + "--commit") + PRINT_COMMIT=1 + ;; + esac + shift +done + + +string_join (){ + local IFS="$1" + shift + echo "$*" +} + +if [[ "$PRINT_VERSION" == "1" ]] || [[ "$PRINT_COMMIT" == "1" ]]; then + DESCRIBE=( $(git describe --always --tags --match 'v*' --dirty --long | tr '-' '\n') ) + + FIELDS=${#DESCRIBE[@]} + if [[ "${DESCRIBE[$FIELDS-1]}" == "dirty" ]]; then + DIRTY=1 + FIELDS=$FIELDS-1 + fi + COMMIT_HASH=${DESCRIBE[$FIELDS-1]} + COMMIT_NO=${DESCRIBE[$FIELDS-2]} + VERSION=$(string_join - ${DESCRIBE[@]:0:$FIELDS-2}) + VERSION=${VERSION:1} + HASH=${DESCRIBE[$FIELDS-1]} + COMMIT=$(printf "c%s.%s" ${DESCRIBE[$FIELDS-2]} ${HASH:1}) + + if [[ "$DIRTY" == "1" ]]; then + COMMIT="$COMMIT.dirty" + fi + +fi + +if [[ "$PRINT_VERSION" == "1" ]]; then + printf $VERSION +fi + +if [[ "$PRINT_VERSION" == "1" ]] && [[ "$PRINT_COMMIT" == "1" ]]; then + printf "+" +fi + +if [[ "$PRINT_COMMIT" == "1" ]]; then + printf $COMMIT +fi + +echo + diff --git a/core/commands/version.go b/core/commands/version.go index c8d3212e2e6..9338ddf2770 100644 --- a/core/commands/version.go +++ b/core/commands/version.go @@ -53,7 +53,7 @@ var VersionCmd = &cmds.Command{ return nil, err } if found && commit { - commitTxt = "-" + v.Commit + commitTxt = "+" + v.Commit } number, found, err := res.Request().Option("number").Bool() diff --git a/repo/config/version.go b/repo/config/version.go index 3acab699bfb..c0e515c7add 100644 --- a/repo/config/version.go +++ b/repo/config/version.go @@ -7,13 +7,15 @@ import ( "time" ) +// CurrentVersionNumber and CurrentCommit are normally set via ldflags -X + // CurrentCommit is the current git commit, this is set as a ldflag in the Makefile var CurrentCommit string // CurrentVersionNumber is the current application's version literal -const CurrentVersionNumber = "0.4.0-dev" +var CurrentVersionNumber = "0.0.0-dev" -const ApiVersion = "/go-ipfs/" + CurrentVersionNumber + "/" +var ApiVersion = "/go-ipfs/" + CurrentVersionNumber + "/" // Version regulates checking if the most recent version is run type Version struct { diff --git a/test/Dockerfile b/test/Dockerfile index 61a4a2c1f90..48760abc9bc 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -42,12 +42,17 @@ RUN apk add --update musl go=$GO_VERSION git bash wget ca-certificates \ COPY . $SRC_PATH +# Change to ARG once travis gets updated docker (1.9+) +ENV IPFS_VERSION=0.4.0-rc2 + RUN cd $SRC_PATH \ && ref="$(cat .git/HEAD | cut -d' ' -f2)" \ && commit="$(cat .git/$ref | head -c 7)" \ - && echo "ldflags=-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$commit" \ + && ldflags="-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$commit \ + -X github.com/ipfs/go-ipfs/repo/config.CurrentVersionNumber=$IPFS_VERSION" \ + && echo "ldflags: $ldflags" \ && cd $SRC_PATH/cmd/ipfs \ - && go build -ldflags "-X github.com/ipfs/go-ipfs/repo/config.CurrentCommit=$commit" \ + && go build -ldflags "$ldflags" \ && cp ipfs /usr/local/bin/ipfs \ && cp $SRC_PATH/bin/container_daemon /usr/local/bin/start_ipfs \ && chmod 755 /usr/local/bin/start_ipfs \ From 595db0a22720e713725229897fe7d6135fb1bf5f Mon Sep 17 00:00:00 2001 From: hutenosa Date: Wed, 9 Mar 2016 16:24:07 +0100 Subject: [PATCH 07/21] various bugfixes fixed bug in main Makefile added make target 'docker' travis docker workaround License: MIT Signed-off-by: hutenosa --- Dockerfile | 5 ++--- Makefile | 5 ++++- test/Dockerfile | 3 +-- test/ipfs-test-lib.sh | 12 +++++++++++- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 01388a7282d..87bd52c965c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,8 +32,7 @@ ENV GOPATH /go ENV PATH /go/bin:$PATH ENV SRC_PATH /go/src/github.com/ipfs/go-ipfs -# Change to ARG once travis gets updated docker (1.9+) -ENV IPFS_VERSION=0.4.0-rc2 +ARG IPFS_VERSION # Get the go-ipfs sourcecode COPY . $SRC_PATH @@ -60,7 +59,7 @@ RUN apk add --update musl go=$GO_VERSION git bash wget ca-certificates \ -X github.com/ipfs/go-ipfs/repo/config.CurrentVersionNumber=$IPFS_VERSION" \ && echo "ldflags: $ldflags" \ && cd $SRC_PATH/cmd/ipfs \ - && go build -ldflags $ldflags \ + && go build -ldflags "$ldflags" \ && cp ipfs /usr/local/bin/ipfs \ && cp $SRC_PATH/bin/container_daemon /usr/local/bin/start_ipfs \ && chmod 755 /usr/local/bin/start_ipfs \ diff --git a/Makefile b/Makefile index 04d69d84a7b..d34930c80ed 100644 --- a/Makefile +++ b/Makefile @@ -62,8 +62,11 @@ clean: uninstall: cd cmd/ipfs && go clean -i -ldflags=$(ldflags) +docker: + docker build --build-arg IPFS_VERSION=$(VERSION) . + PHONY += all help godep toolkit_upgrade gx_upgrade gxgo_upgrade gx_check -PHONY += go_check deps vendor install build nofuse clean uninstall +PHONY += go_check deps vendor install build nofuse clean uninstall docker ############################################################## # tests targets diff --git a/test/Dockerfile b/test/Dockerfile index 48760abc9bc..3ce2971ea50 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -42,8 +42,7 @@ RUN apk add --update musl go=$GO_VERSION git bash wget ca-certificates \ COPY . $SRC_PATH -# Change to ARG once travis gets updated docker (1.9+) -ENV IPFS_VERSION=0.4.0-rc2 +ARG IPFS_VERSION RUN cd $SRC_PATH \ && ref="$(cat .git/HEAD | cut -d' ' -f2)" \ diff --git a/test/ipfs-test-lib.sh b/test/ipfs-test-lib.sh index 94ed5a77ff0..504381e986b 100644 --- a/test/ipfs-test-lib.sh +++ b/test/ipfs-test-lib.sh @@ -48,7 +48,17 @@ shellquote() { # This takes a Dockerfile, and a build context directory docker_build() { - docker build --rm -f "$1" "$2" + ipfs_version=$($2/bin/genversion --version) + + # Workaround for docker < 1.9 + if test "$TRAVIS" = true + then + cat "$1" | sed "s/^ARG IPFS_VERSION.*/ENV IPFS_VERSION=$ipfs_version/" > "$1.travis" + docker build --rm -f "$1.travis" "$2" + rm "$1.travis" + else + docker build --rm --build-arg IPFS_VERSION=$ipfs_version -f "$1" "$2" + fi } # This takes an image as argument and writes a docker ID on stdout From d801991bb6c600c05aaba13963be008e4bcce852 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 14 Mar 2016 20:18:39 +0100 Subject: [PATCH 08/21] feat: Update the webui to work with the latest changes in 0.4 License: MIT Signed-off-by: dignifiedquire --- core/corehttp/webui.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/corehttp/webui.go b/core/corehttp/webui.go index db08731a983..8d4445ef3cc 100644 --- a/core/corehttp/webui.go +++ b/core/corehttp/webui.go @@ -1,7 +1,7 @@ package corehttp // TODO: move to IPNS -const WebUIPath = "/ipfs/QmRyWyKWmphamkMRnJVjUTzSFSAAZowYP4rnbgnfMXC9Mr" +const WebUIPath = "/ipfs/QmU3o9bvfenhTKhxUakbYrLDnZU7HezAVxPM6Ehjw9Xjqy" // this is a list of all past webUI paths. var WebUIPaths = []string{ @@ -13,6 +13,7 @@ var WebUIPaths = []string{ "/ipfs/QmctngrQAt9fjpQUZr7Bx3BsXUcif52eZGTizWhvcShsjz", "/ipfs/QmS2HL9v5YeKgQkkWMvs1EMnFtUowTEdFfSSeMT4pos1e6", "/ipfs/QmR9MzChjp1MdFWik7NjEjqKQMzVmBkdK3dz14A6B5Cupm", + "/ipfs/QmRyWyKWmphamkMRnJVjUTzSFSAAZowYP4rnbgnfMXC9Mr", } var WebUIOption = RedirectOption("webui", WebUIPath) From aeb1ed2b0b30d9622f84a4e4a8a50fc9454cc35c Mon Sep 17 00:00:00 2001 From: Jeromy Johnson Date: Mon, 14 Mar 2016 16:11:29 -0700 Subject: [PATCH 09/21] fix double transfer encoding head problem License: MIT Signed-off-by: Jeromy Johnson --- commands/http/client.go | 2 +- commands/http/handler.go | 34 ++++++++++--------- commands/http/handler_test.go | 2 -- test/sharness/t0110-gateway.sh | 10 ++++++ ...230-channel-streaming-http-content-type.sh | 10 +++--- 5 files changed, 33 insertions(+), 25 deletions(-) diff --git a/commands/http/client.go b/commands/http/client.go index eec01a26826..5db1ec05192 100644 --- a/commands/http/client.go +++ b/commands/http/client.go @@ -159,7 +159,7 @@ func getResponse(httpRes *http.Response, req cmds.Request) (cmds.Response, error contentType := httpRes.Header.Get(contentTypeHeader) contentType = strings.Split(contentType, ";")[0] - lengthHeader := httpRes.Header.Get(contentLengthHeader) + lengthHeader := httpRes.Header.Get(extraContentLengthHeader) if len(lengthHeader) > 0 { length, err := strconv.ParseUint(lengthHeader, 10, 64) if err != nil { diff --git a/commands/http/handler.go b/commands/http/handler.go index 9ac6d4a4a43..82850764ff3 100644 --- a/commands/http/handler.go +++ b/commands/http/handler.go @@ -41,20 +41,23 @@ var ( ) const ( - StreamErrHeader = "X-Stream-Error" - streamHeader = "X-Stream-Output" - channelHeader = "X-Chunked-Output" - uaHeader = "User-Agent" - contentTypeHeader = "Content-Type" - contentLengthHeader = "Content-Length" - contentDispHeader = "Content-Disposition" - transferEncodingHeader = "Transfer-Encoding" - applicationJson = "application/json" - applicationOctetStream = "application/octet-stream" - plainText = "text/plain" - originHeader = "origin" + StreamErrHeader = "X-Stream-Error" + streamHeader = "X-Stream-Output" + channelHeader = "X-Chunked-Output" + extraContentLengthHeader = "X-Content-Length" + uaHeader = "User-Agent" + contentTypeHeader = "Content-Type" + contentDispHeader = "Content-Disposition" + transferEncodingHeader = "Transfer-Encoding" + applicationJson = "application/json" + applicationOctetStream = "application/octet-stream" + plainText = "text/plain" + originHeader = "origin" ) +var AllowedExposedHeadersArr = []string{streamHeader, channelHeader, extraContentLengthHeader} +var AllowedExposedHeaders = strings.Join(AllowedExposedHeadersArr, ", ") + const ( ACAOrigin = "Access-Control-Allow-Origin" ACAMethods = "Access-Control-Allow-Methods" @@ -241,7 +244,7 @@ func sendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req h.Set("Trailer", StreamErrHeader) if res.Length() > 0 { - h.Set(contentLengthHeader, strconv.FormatUint(res.Length(), 10)) + h.Set("X-Content-Length", strconv.FormatUint(res.Length(), 10)) } if _, ok := res.Output().(io.Reader); ok { @@ -268,12 +271,11 @@ func sendResponse(w http.ResponseWriter, r *http.Request, res cmds.Response, req } h.Set(contentTypeHeader, mime) - h.Set(transferEncodingHeader, "chunked") // set 'allowed' headers - h.Set("Access-Control-Allow-Headers", "X-Stream-Output, X-Chunked-Output") + h.Set("Access-Control-Allow-Headers", AllowedExposedHeaders) // expose those headers - h.Set("Access-Control-Expose-Headers", "X-Stream-Output, X-Chunked-Output") + h.Set("Access-Control-Expose-Headers", AllowedExposedHeaders) if r.Method == "HEAD" { // after all the headers. return diff --git a/commands/http/handler_test.go b/commands/http/handler_test.go index e40fb72b5a7..d31e54e3125 100644 --- a/commands/http/handler_test.go +++ b/commands/http/handler_test.go @@ -11,8 +11,6 @@ import ( coremock "github.com/ipfs/go-ipfs/core/mock" ) -const AllowedExposedHeaders = "X-Stream-Output, X-Chunked-Output" - func assertHeaders(t *testing.T, resHeaders http.Header, reqHeaders map[string]string) { for name, value := range reqHeaders { if resHeaders.Get(name) != value { diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh index dacf4c4b8e3..46b25b7fa35 100755 --- a/test/sharness/t0110-gateway.sh +++ b/test/sharness/t0110-gateway.sh @@ -91,6 +91,16 @@ test_expect_success "log output looks good" ' grep "log API client connected" log_out ' +test_expect_success "GET /api/v0/version succeeds" ' + curl -v "http://127.0.0.1:$apiport/api/v0/version" 2> version_out +' + +test_expect_success "output only has one transfer encoding header" ' + grep "Transfer-Encoding: chunked" version_out | wc -l > tecount_out && + echo " 1" > tecount_exp && + test_cmp tecount_out tecount_exp +' + test_expect_success "setup index hash" ' mkdir index && echo "

" > index/index.html && diff --git a/test/sharness/t0230-channel-streaming-http-content-type.sh b/test/sharness/t0230-channel-streaming-http-content-type.sh index f6e5a6a7f29..2a95695e36e 100755 --- a/test/sharness/t0230-channel-streaming-http-content-type.sh +++ b/test/sharness/t0230-channel-streaming-http-content-type.sh @@ -21,12 +21,11 @@ test_ls_cmd() { test_expect_success "Text encoded channel-streaming command output looks good" ' printf "HTTP/1.1 200 OK\r\n" >expected_output && - printf "Access-Control-Allow-Headers: X-Stream-Output, X-Chunked-Output\r\n" >>expected_output && - printf "Access-Control-Expose-Headers: X-Stream-Output, X-Chunked-Output\r\n" >>expected_output && + printf "Access-Control-Allow-Headers: X-Stream-Output, X-Chunked-Output, X-Content-Length\r\n" >>expected_output && + printf "Access-Control-Expose-Headers: X-Stream-Output, X-Chunked-Output, X-Content-Length\r\n" >>expected_output && printf "Content-Type: text/plain\r\n" >>expected_output && printf "Server: go-ipfs/%s\r\n" $(ipfs version -n) >>expected_output && printf "Trailer: X-Stream-Error\r\n" >>expected_output && - printf "Transfer-Encoding: chunked\r\n" >>expected_output && printf "X-Chunked-Output: 1\r\n" >>expected_output && printf "Transfer-Encoding: chunked\r\n" >>expected_output && printf "\r\n" >>expected_output && @@ -44,12 +43,11 @@ test_ls_cmd() { test_expect_success "JSON encoded channel-streaming command output looks good" ' printf "HTTP/1.1 200 OK\r\n" >expected_output && - printf "Access-Control-Allow-Headers: X-Stream-Output, X-Chunked-Output\r\n" >>expected_output && - printf "Access-Control-Expose-Headers: X-Stream-Output, X-Chunked-Output\r\n" >>expected_output && + printf "Access-Control-Allow-Headers: X-Stream-Output, X-Chunked-Output, X-Content-Length\r\n" >>expected_output && + printf "Access-Control-Expose-Headers: X-Stream-Output, X-Chunked-Output, X-Content-Length\r\n" >>expected_output && printf "Content-Type: application/json\r\n" >>expected_output && printf "Server: go-ipfs/%s\r\n" $(ipfs version -n) >>expected_output && printf "Trailer: X-Stream-Error\r\n" >>expected_output && - printf "Transfer-Encoding: chunked\r\n" >>expected_output && printf "X-Chunked-Output: 1\r\n" >>expected_output && printf "Transfer-Encoding: chunked\r\n" >>expected_output && printf "\r\n" >>expected_output && From 22937ad2d4dda68b0776c031bd8da58057c6d70e Mon Sep 17 00:00:00 2001 From: Jeromy Johnson Date: Mon, 14 Mar 2016 16:36:31 -0700 Subject: [PATCH 10/21] fix whitespace trimming License: MIT Signed-off-by: Jeromy Johnson --- test/sharness/t0110-gateway.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh index 46b25b7fa35..59887c73498 100755 --- a/test/sharness/t0110-gateway.sh +++ b/test/sharness/t0110-gateway.sh @@ -96,8 +96,8 @@ test_expect_success "GET /api/v0/version succeeds" ' ' test_expect_success "output only has one transfer encoding header" ' - grep "Transfer-Encoding: chunked" version_out | wc -l > tecount_out && - echo " 1" > tecount_exp && + grep "Transfer-Encoding: chunked" version_out | wc -l | xargs echo > tecount_out && + echo "1" > tecount_exp && test_cmp tecount_out tecount_exp ' From d5169e4eeb2f7b654cdff58687abe19503c1e74e Mon Sep 17 00:00:00 2001 From: Richard Littauer Date: Tue, 15 Mar 2016 13:23:22 -0400 Subject: [PATCH 11/21] Alphabetized subcommands License: MIT Signed-off-by: Richard Littauer --- core/commands/object/object.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/core/commands/object/object.go b/core/commands/object/object.go index 86091ad0727..b2f13d01f6d 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -47,24 +47,24 @@ var ObjectCmd = &cmds.Command{ 'ipfs object' is a plumbing command used to manipulate DAG objects directly.`, Synopsis: ` -ipfs object get - Get the DAG node named by -ipfs object put - Stores input, outputs its key ipfs object data - Outputs raw bytes in an object +ipfs object get - Get the DAG node named by ipfs object links - Outputs links pointed to by object -ipfs object stat - Outputs statistics of object ipfs object new