Add static json file caching

Avoid the call gocode during this condition - Conditions - Hook the insert of dot(.) - Typed the package has not yet been imported use context['input'] - Match the typed package name and json file name Signed-off-by: Koichi Shiraishi <k@zchee.io>
deoplete-plugins · Apr 26, 2016 · 03d4dc2 · 03d4dc2
1 parent f1ab54b
commit 03d4dc2
Show file tree

Hide file tree

Showing 278 changed files with 836 additions and 175 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,8 @@
+./benchmark
+./build
+./images
+./plugin
+./*.tags
+
+# json tarballs
+*.tar.gz
diff --git a/.gitignore b/.gitignore
@@ -57,3 +57,7 @@ docs/_build/
 
 # PyBuilder
 target/
+
+
+# json tarballs
+*.tar.gz
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,20 @@
+FROM golang:1.6.2-alpine
+MAINTAINER zchee <k@zchee.io>
+
+RUN set -ex \
+	&& apk add --no-cache --virtual .build-deps \
+		make \
+		git \
+		tar \
+		python3 \
+	\
+	&& go get -u -v github.com/nsf/gocode
+
+COPY ./ /deoplete-go
+
+RUN cd /deoplete-go \
+	&& make gen_json \
+	\
+	&& tar cf json_1.6.2_linux_amd64.tar.gz ./data/json/1.6.2/linux_amd64
+
+CMD ["cat", "/deoplete-go/json_1.6.2_linux_amd64.tar.gz"]
diff --git a/Makefile b/Makefile
@@ -3,11 +3,23 @@ TARGET = ./rplugin/python3/deoplete/ujson.so
 CURRENT := $(shell pwd)
 RPLUGIN_HOME := $(CURRENT)/rplugin/python3
 PYTHON3 := $(shell which python3)
+DOCKER := $(shell which docker)
+DOKCER_IMAGE := zchee/deoplete-go:linux_amd64
 GIT := $(shell which git)
+GOCODE := $(shell which gocode)
+GOVERSION = $(shell go version | awk '{print $$3}' | sed -e 's/go//')
+GOOS := $(shell go env GOOS)
+GOARCH := $(shell go env GOARCH)
 
 RPLUGIN_PATH := ./rplugin/python3/deoplete/sources/
 MODULE_NAME := deoplete_go.py
 
+PACKAGE ?= unsafe
+
+ifneq ($(PACKAGE),unsafe)
+	PACKAGE += unsafe
+endif
+
 
 all : $(TARGET)
 
@@ -18,6 +30,28 @@ build/:
 rplugin/python3/deoplete/ujson.so: build/
 	cp $(shell find $(CURRENT)/build -name ujson*.so) $(RPLUGIN_HOME)/deoplete/ujson.so
 
+data/stdlib.txt:
+	go tool api -contexts $(GOOS)-$(GOARCH)-cgo | sed -e s/,//g | awk '{print $$2}' | uniq > ./data/stdlib.txt
+	@for pkg in $(PACKAGE) ; do \
+		echo $$pkg >> ./data/stdlib.txt; \
+	done
+	mv ./data/stdlib.txt ./data/stdlib-$(GOVERSION)_$(GOOS)_$(GOARCH).txt
+
+gen_json: data/stdlib.txt
+	$(GOCODE) close
+	cd ./data && ./gen_json.py $(GOOS) $(GOARCH)
+
+docker_build:
+	$(DOCKER) build -t $(DOKCER_IMAGE) .
+
+docker_gen_json: docker_build
+	$(DOCKER) run --rm $(DOKCER_IMAGE) > ./json_1.6.2_linux_amd64.tar.gz
+	tar xf ./json_1.6.2_linux_amd64.tar.gz
+	mv ./json_1.6.2_linux_amd64.tar.gz ./data/json_1.6_linux_amd64.tar.gz
+
+docker_gen_stdlib: docker_build
+	$(DOCKER) run --rm $(DOKCER_IMAGE) cat /deoplete-go/data/stdlib-1.6.2_linux_amd64.txt > ./data/stdlib-1.6.2_linux_amd64.txt
+
 test: lint
 
 lint: flake8

diff --git a/README.md b/README.md
@@ -88,12 +88,14 @@ Plug 'zchee/deoplete-go', { 'do': 'make'}
 
 ## Available Settings
 
-| Setting value                         | Default | Required  |
-|:--------------------------------------|:-------:|:---------:|
-| `g:deoplete#sources#go#align_class`   | `0`     | No        |
-| `g:deoplete#sources#go#gocode_binary` | `''`    | Recommend |
-| `g:deoplete#sources#go#package_dot`   | `0`     | No        |
-| `g:deoplete#sources#go#sort_class`    | `[]`    | Recommend |
+| Setting value                          | Default                        | Required  |
+|:---------------------------------------|:------------------------------:|:---------:|
+| `g:deoplete#sources#go#align_class`    | `0`                            | No        |
+| `g:deoplete#sources#go#gocode_binary`  | `''`                           | Recommend |
+| `g:deoplete#sources#go#package_dot`    | `0`                            | No        |
+| `g:deoplete#sources#go#sort_class`     | `[]`                           | Recommend |
+| `g:deoplete#sources#go#use_cache`      | `0`                            | Recommend |
+| `g:deoplete#sources#go#data_directory` | `$HOME.'/.config/gocode/json'` | Recommend |
 
 ### `g:deoplete#sources#go#align_class`
 #### Class Aligning
@@ -171,6 +173,71 @@ let g:deoplete#sources#go#sort_class = ['package', 'func', 'type', 'var', 'const
 
 Try test it with the `os` package :)
 
+### `g:deoplete#sources#go#use_cache` `g:deoplete#sources#go#data_directory`
+#### Static json caching
+
+`g:deoplete#sources#go#use_cache`
+
+| **Default**  | `0`           |
+|--------------|---------------|
+| **Required** | **Recommend** |
+| **Type**     | int           |
+| **Example**  | `1`           |
+
+`g:deoplete#sources#go#data_directory`
+
+| **Default**  | `~/.cache/gocode/json` |
+|--------------|------------------------|
+| **Required** | **Recommend**          |
+| **Type**     | string                 |
+| **Example**  | `'/path/to/data_dir'`  |
+
+Use static json caching Go stdlib package API.
+If matched name of stdlib and input package name, it returns the static json data without the use of gocode.
+and, Possible to get package API if have not `import` of current buffer.
+
+Terms:
+
+- Hook the insert of dot `.` (e.g. `fmt.|`)
+- You typed package name have not `import` current buffer
+- Match the typed package name and json file name
+
+`deoplete-go` will parse `g:deoplete#sources#go#data_directory` directory. You can define of json data directory.
+Default is `~/.cache/gocode/json`.
+
+Also, See [How to use static json caching](#how-to-use-static-json-caching)
+
+```vim
+let g:deoplete#sources#go#use_cache = 1
+let g:deoplete#sources#go#data_directory = '/path/to/data_dir'
+```
+
+===
+
+### How to use static json caching
+
+| **Current Go version** | `1.6`                 |
+|------------------------|-----------------------|
+| `$GOOS`                | `darwin`, `linux`     |
+| `$GOARCH`              | `amd64`               |
+
+Pre-generate json data is [data/json](./data/json).
+If you use it, `cp -r data/json/VERSION/$GOOS_$GOARCH /path/to/data_dir`.
+`/path/to/data_dir` is `g:deoplete#sources#go#data_directory`.
+
+And, You can generate your Go environment. such as version is `devel`, GOARCH is `arm`.
+If you want to it, run `make gen_json`.
+Will generated json file to `./data/json/VERSION/$GOOS_$GOARCH`.
+
+`make gen_json` command also will create `./data/stdlib.txt`. It same as `go tool api` result.
+In detail,
+```bash
+go tool api -contexts $GOOS-$GOARCH-cgo | grep -v 'golang.org/x/net/http2/hpack' | sed -e s/,//g | awk '{print $2}' | uniq > ./data/stdlib.txt
+```
+
+This api list used in the base for the generation of json file.
+
+
 ===
 
 ## Sample init.vim
@@ -191,6 +258,8 @@ let g:deoplete#enable_at_startup = 1
 " deoplete-go settings
 let g:deoplete#sources#go#gocode_binary = $GOPATH.'/bin/gocode'
 let g:deoplete#sources#go#sort_class = ['package', 'func', 'type', 'var', 'const']
+let g:deoplete#sources#go#use_cache = 1
+let g:deoplete#sources#go#data_directory = '/path/to/data_dir'
 ```
 
 ===
@@ -199,7 +268,7 @@ TODO:
 -----
 - [ ] Parse included cgo (C, C++ language) headers on current buffer
   - `ctags` will be blocking `deoplete.nvim`
-- [ ] Support static json caching
+- [x] Support static json caching
   - See https://github.com/zchee/deoplete-go/pull/19
 - [x] Support Go stdlib package `import "***"` name completion
   - This feature has been implemented in gocode. Thanks @nhooyr!

diff --git a/data/gen_json.py b/data/gen_json.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+import json
+import os
+import subprocess
+import sys
+import re
+
+def main():
+    base_offset = 41
+
+    go_version_raw = subprocess.Popen("go version",
+                                  shell=True,
+                                  stdout=subprocess.PIPE
+                                  ).stdout.read()
+
+    go_version = str(go_version_raw).split(' ')[2].strip('go')
+    go_os = sys.argv[1]
+    go_arch = sys.argv[2]
+
+    with open('./stdlib-' + go_version + '_' + go_os + '_' + go_arch + '.txt') as stdlib:
+        packages = stdlib.read().splitlines()
+
+    for pkg in packages:
+        template_file = './template.go'
+        f = open(template_file)
+        fs = f.read(-1)
+
+        func = None
+        if re.search(r'/', pkg):
+            func = str(pkg).split(r'/')[-1]
+        else:
+            func = pkg
+        source = str(fs).replace('IMPORT', pkg).replace('FUNC', func).encode()
+
+        offset = base_offset + (len(pkg) + len(func))
+
+        process = subprocess.Popen([FindBinaryPath('gocode'),
+                                    '-f=json',
+                                    'autocomplete',
+                                    template_file,
+                                    str(offset)],
+                                   stdin=subprocess.PIPE,
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE,
+                                   start_new_session=True)
+        process.stdin.write(source)
+        stdout_data, stderr_data = process.communicate()
+        result = json.loads(stdout_data.decode())
+
+        out_dir = os.path.join(
+            './json', go_version, go_os + '_' + go_arch)
+        if not os.path.exists(out_dir):
+            os.makedirs(out_dir)
+
+        out_path = \
+            os.path.join(out_dir, func + '.json')
+        out = open(out_path, 'w')
+        out.write(json.dumps(result, sort_keys=True))
+        out.close()
+
+        print(pkg)
+
+
+def FindBinaryPath(cmd):
+    def is_exec(fpath):
+        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+    fpath, fname = os.path.split(cmd)
+    if fpath:
+        if is_exec(cmd):
+            return cmd
+    else:
+        for path in os.environ["PATH"].split(os.pathsep):
+            path = path.strip('"')
+            binary = os.path.join(path, cmd)
+            if is_exec(binary):
+                return binary
+    return error(self.vim, 'gocode binary not found')
+
+main()
diff --git a/data/json/1.6.2/darwin_amd64/adler32.json b/data/json/1.6.2/darwin_amd64/adler32.json
@@ -0,0 +1 @@
+[0, [{"class": "const", "name": "Size", "type": ""}, {"class": "func", "name": "Checksum", "type": "func(data []byte) uint32"}, {"class": "func", "name": "New", "type": "func() hash.Hash32"}]]
diff --git a/data/json/1.6.2/darwin_amd64/aes.json b/data/json/1.6.2/darwin_amd64/aes.json
@@ -0,0 +1 @@
+[0, [{"class": "const", "name": "BlockSize", "type": ""}, {"class": "func", "name": "NewCipher", "type": "func(key []byte) (cipher.Block, error)"}, {"class": "type", "name": "KeySizeError", "type": "int"}]]
diff --git a/data/json/1.6.2/darwin_amd64/ascii85.json b/data/json/1.6.2/darwin_amd64/ascii85.json
@@ -0,0 +1 @@
+[0, [{"class": "func", "name": "Decode", "type": "func(dst []byte, src []byte, flush bool) (ndst int, nsrc int, err error)"}, {"class": "func", "name": "Encode", "type": "func(dst []byte, src []byte) int"}, {"class": "func", "name": "MaxEncodedLen", "type": "func(n int) int"}, {"class": "func", "name": "NewDecoder", "type": "func(r io.Reader) io.Reader"}, {"class": "func", "name": "NewEncoder", "type": "func(w io.Writer) io.WriteCloser"}, {"class": "type", "name": "CorruptInputError", "type": "int64"}]]
diff --git a/data/json/1.6.2/darwin_amd64/asn1.json b/data/json/1.6.2/darwin_amd64/asn1.json
@@ -0,0 +1 @@
+[0, [{"class": "const", "name": "ClassApplication", "type": ""}, {"class": "const", "name": "ClassContextSpecific", "type": ""}, {"class": "const", "name": "ClassPrivate", "type": ""}, {"class": "const", "name": "ClassUniversal", "type": ""}, {"class": "const", "name": "TagBitString", "type": ""}, {"class": "const", "name": "TagBoolean", "type": ""}, {"class": "const", "name": "TagEnum", "type": ""}, {"class": "const", "name": "TagGeneralString", "type": ""}, {"class": "const", "name": "TagGeneralizedTime", "type": ""}, {"class": "const", "name": "TagIA5String", "type": ""}, {"class": "const", "name": "TagInteger", "type": ""}, {"class": "const", "name": "TagOID", "type": ""}, {"class": "const", "name": "TagOctetString", "type": ""}, {"class": "const", "name": "TagPrintableString", "type": ""}, {"class": "const", "name": "TagSequence", "type": ""}, {"class": "const", "name": "TagSet", "type": ""}, {"class": "const", "name": "TagT61String", "type": ""}, {"class": "const", "name": "TagUTCTime", "type": ""}, {"class": "const", "name": "TagUTF8String", "type": ""}, {"class": "func", "name": "Marshal", "type": "func(val interface{}) ([]byte, error)"}, {"class": "func", "name": "Unmarshal", "type": "func(b []byte, val interface{}) (rest []byte, err error)"}, {"class": "func", "name": "UnmarshalWithParams", "type": "func(b []byte, val interface{}, params string) (rest []byte, err error)"}, {"class": "type", "name": "BitString", "type": "struct"}, {"class": "type", "name": "Enumerated", "type": "int"}, {"class": "type", "name": "Flag", "type": "bool"}, {"class": "type", "name": "ObjectIdentifier", "type": "[]int"}, {"class": "type", "name": "RawContent", "type": "[]byte"}, {"class": "type", "name": "RawValue", "type": "struct"}, {"class": "type", "name": "StructuralError", "type": "struct"}, {"class": "type", "name": "SyntaxError", "type": "struct"}]]
diff --git a/data/json/1.6.2/darwin_amd64/ast.json b/data/json/1.6.2/darwin_amd64/ast.json
@@ -0,0 +1 @@
+[0, [{"class": "const", "name": "Bad", "type": ""}, {"class": "const", "name": "Con", "type": ""}, {"class": "const", "name": "FilterFuncDuplicates", "type": ""}, {"class": "const", "name": "FilterImportDuplicates", "type": ""}, {"class": "const", "name": "FilterUnassociatedComments", "type": ""}, {"class": "const", "name": "Fun", "type": ""}, {"class": "const", "name": "Lbl", "type": ""}, {"class": "const", "name": "Pkg", "type": ""}, {"class": "const", "name": "RECV", "type": ""}, {"class": "const", "name": "SEND", "type": ""}, {"class": "const", "name": "Typ", "type": ""}, {"class": "const", "name": "Var", "type": ""}, {"class": "func", "name": "FileExports", "type": "func(src *ast.File) bool"}, {"class": "func", "name": "FilterDecl", "type": "func(decl ast.Decl, f ast.Filter) bool"}, {"class": "func", "name": "FilterFile", "type": "func(src *ast.File, f ast.Filter) bool"}, {"class": "func", "name": "FilterPackage", "type": "func(pkg *ast.Package, f ast.Filter) bool"}, {"class": "func", "name": "Fprint", "type": "func(w io.Writer, fset *token.FileSet, x interface{}, f ast.FieldFilter) error"}, {"class": "func", "name": "Inspect", "type": "func(node ast.Node, f func(ast.Node) bool)"}, {"class": "func", "name": "IsExported", "type": "func(name string) bool"}, {"class": "func", "name": "MergePackageFiles", "type": "func(pkg *ast.Package, mode ast.MergeMode) *ast.File"}, {"class": "func", "name": "NewCommentMap", "type": "func(fset *token.FileSet, node ast.Node, comments []*ast.CommentGroup) ast.CommentMap"}, {"class": "func", "name": "NewIdent", "type": "func(name string) *ast.Ident"}, {"class": "func", "name": "NewObj", "type": "func(kind ast.ObjKind, name string) *ast.Object"}, {"class": "func", "name": "NewPackage", "type": "func(fset *token.FileSet, files map[string]*ast.File, importer ast.Importer, universe *ast.Scope) (*ast.Package, error)"}, {"class": "func", "name": "NewScope", "type": "func(outer *ast.Scope) *ast.Scope"}, {"class": "func", "name": "NotNilFilter", "type": "func(_ string, v reflect.Value) bool"}, {"class": "func", "name": "PackageExports", "type": "func(pkg *ast.Package) bool"}, {"class": "func", "name": "Print", "type": "func(fset *token.FileSet, x interface{}) error"}, {"class": "func", "name": "SortImports", "type": "func(fset *token.FileSet, f *ast.File)"}, {"class": "func", "name": "Walk", "type": "func(v ast.Visitor, node ast.Node)"}, {"class": "type", "name": "ArrayType", "type": "struct"}, {"class": "type", "name": "AssignStmt", "type": "struct"}, {"class": "type", "name": "BadDecl", "type": "struct"}, {"class": "type", "name": "BadExpr", "type": "struct"}, {"class": "type", "name": "BadStmt", "type": "struct"}, {"class": "type", "name": "BasicLit", "type": "struct"}, {"class": "type", "name": "BinaryExpr", "type": "struct"}, {"class": "type", "name": "BlockStmt", "type": "struct"}, {"class": "type", "name": "BranchStmt", "type": "struct"}, {"class": "type", "name": "CallExpr", "type": "struct"}, {"class": "type", "name": "CaseClause", "type": "struct"}, {"class": "type", "name": "ChanDir", "type": "int"}, {"class": "type", "name": "ChanType", "type": "struct"}, {"class": "type", "name": "CommClause", "type": "struct"}, {"class": "type", "name": "Comment", "type": "struct"}, {"class": "type", "name": "CommentGroup", "type": "struct"}, {"class": "type", "name": "CommentMap", "type": "map[ast.Node][]*ast.CommentGroup"}, {"class": "type", "name": "CompositeLit", "type": "struct"}, {"class": "type", "name": "Decl", "type": "interface"}, {"class": "type", "name": "DeclStmt", "type": "struct"}, {"class": "type", "name": "DeferStmt", "type": "struct"}, {"class": "type", "name": "Ellipsis", "type": "struct"}, {"class": "type", "name": "EmptyStmt", "type": "struct"}, {"class": "type", "name": "Expr", "type": "interface"}, {"class": "type", "name": "ExprStmt", "type": "struct"}, {"class": "type", "name": "Field", "type": "struct"}, {"class": "type", "name": "FieldFilter", "type": "func(name string, value reflect.Value) bool"}, {"class": "type", "name": "FieldList", "type": "struct"}, {"class": "type", "name": "File", "type": "struct"}, {"class": "type", "name": "Filter", "type": "func(string) bool"}, {"class": "type", "name": "ForStmt", "type": "struct"}, {"class": "type", "name": "FuncDecl", "type": "struct"}, {"class": "type", "name": "FuncLit", "type": "struct"}, {"class": "type", "name": "FuncType", "type": "struct"}, {"class": "type", "name": "GenDecl", "type": "struct"}, {"class": "type", "name": "GoStmt", "type": "struct"}, {"class": "type", "name": "Ident", "type": "struct"}, {"class": "type", "name": "IfStmt", "type": "struct"}, {"class": "type", "name": "ImportSpec", "type": "struct"}, {"class": "type", "name": "Importer", "type": "func(imports map[string]*ast.Object, path string) (pkg *ast.Object, err error)"}, {"class": "type", "name": "IncDecStmt", "type": "struct"}, {"class": "type", "name": "IndexExpr", "type": "struct"}, {"class": "type", "name": "InterfaceType", "type": "struct"}, {"class": "type", "name": "KeyValueExpr", "type": "struct"}, {"class": "type", "name": "LabeledStmt", "type": "struct"}, {"class": "type", "name": "MapType", "type": "struct"}, {"class": "type", "name": "MergeMode", "type": "uint"}, {"class": "type", "name": "Node", "type": "interface"}, {"class": "type", "name": "ObjKind", "type": "int"}, {"class": "type", "name": "Object", "type": "struct"}, {"class": "type", "name": "Package", "type": "struct"}, {"class": "type", "name": "ParenExpr", "type": "struct"}, {"class": "type", "name": "RangeStmt", "type": "struct"}, {"class": "type", "name": "ReturnStmt", "type": "struct"}, {"class": "type", "name": "Scope", "type": "struct"}, {"class": "type", "name": "SelectStmt", "type": "struct"}, {"class": "type", "name": "SelectorExpr", "type": "struct"}, {"class": "type", "name": "SendStmt", "type": "struct"}, {"class": "type", "name": "SliceExpr", "type": "struct"}, {"class": "type", "name": "Spec", "type": "interface"}, {"class": "type", "name": "StarExpr", "type": "struct"}, {"class": "type", "name": "Stmt", "type": "interface"}, {"class": "type", "name": "StructType", "type": "struct"}, {"class": "type", "name": "SwitchStmt", "type": "struct"}, {"class": "type", "name": "TypeAssertExpr", "type": "struct"}, {"class": "type", "name": "TypeSpec", "type": "struct"}, {"class": "type", "name": "TypeSwitchStmt", "type": "struct"}, {"class": "type", "name": "UnaryExpr", "type": "struct"}, {"class": "type", "name": "ValueSpec", "type": "struct"}, {"class": "type", "name": "Visitor", "type": "interface"}]]