Sby10 (#175)

* Added allowed request headers, decryption, and log optimization * update readme * update env name * update readme
CESSProject · Aug 14, 2024 · a6e541a · a6e541a
1 parent 3fbfb72
commit a6e541a
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 306 deletions.
diff --git a/.env.local b/.env.local
@@ -19,7 +19,7 @@ rpc="wss://testnet-rpc.cess.network/ws/"
 
 
 # communication ports in the storage network
-sport="4001"
+storage_port="4001"
 # bootstrap nodes in the storage network, multiple separated by spaces
 boot="_dnsaddr.boot-miner-testnet.cess.network"
 

diff --git a/README.md b/README.md
@@ -86,10 +86,7 @@ service iptables restart
 
 ## 🏗 Get the binary program
 ### Method one
-Download the latest release of the binary application directly at：
-```
-wget https://github.com/CESSProject/DeOSS/releases/download/v0.3.7/DeOSS0.3.7.linux-amd64.tar.gz
-```
+Download the latest release code directly at： [Releases](https://github.com/CESSProject/DeOSS/releases)
 
 ### Method two
 Compile the binary program from the DeOSS source code and follow the process as follows:
@@ -243,273 +240,7 @@ It is generally not recommended to use this command：
 
 # 📖 Usage for API
 
-The public API endpoint URL of DeOSS is the server you deploy, All endpoints described in this document should be made relative to this root URL,The following example uses URL instead.
-
-**Before using DeOSS, you must authorize it as follows:** 
-
-1. Create a wallet account and fund it, refer to [Configure Wallet](https://github.com/CESSProject/DeOSS#configure-wallet)
-
-2. [Purchase a territory](https://github.com/CESSProject/doc-v2/blob/main/products/deoss/picture/buy_territory.png)
-
-3. Authorize the use right to DeOSS:[Authorize](https://github.com/CESSProject/W3F-illustration/blob/4995c1584006823990806b9d30fa7d554630ec14/deoss/authorizeOss.png)
-
-
-## Identity signature
-
-Calling some APIs requires authentication of your identity. In web3, your wallet is your identity. Generate your signature data in [the block browser](https://polkadot.js.org/apps/), and then add your signature information in the API request header to authenticate your identity. Please refer to [the signature method](https://github.com/CESSProject/doc-v2/blob/main/products/deoss/picture/sign.png).
-
-The authentication information you need to add in the header:
-
-| Key       | Description    | Example |
-| --------- | -------------- | ------- |
-| Account   | wallet account | cX...   |
-| Message   | signed message | ...     |
-| Signature | signature      | 0x...   |
-
-
-## Create a bucket
-
-| **PUT**  /bucket |
-| ---------------- |
-
-The put bucket interface is used to create a bucket. When uploading files, the bucket must be specified for storage.
-
-- Request Header
-
-| key           | value               |
-| ------------- | ------------------- |
-| Bucket        | created bucket name |
-
-_Identity signature required: yes_
-
-- Request example
-
-```shell
-# curl -X PUT URL/ -H "Bucket: bucket_name" -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..."
-```
-
-## Upload a file
-
-| **PUT**  /file |
-| -------------- |
-
-The put file interface is used to upload files to the cess system. You need to submit the file as form data and use provide the specific field.
-If the upload is successful, you will get the fid of the file. If you want to encrypt your file, you can specify the `cipher` field in the header and enter your password (the length cannot exceed 32 characters), and the system will automatically encrypt it.
-
-- Request Header
-
-| key              | description        |
-| ---------------- | ------------------ |
-| Bucket           | bucket name        |
-| Territory        | territory name     |
-| Cipher(optional) | cipher             |
-
-_Identity signature required: yes_
-
-- Request Body
-
-| key  | value        |
-| ---- | ------------ |
-| file | file[binary] |
-
-- Request example
-
-```shell
-# curl -X PUT URL/file -F 'file=@test.log;type=application/octet-stream' -H "Bucket: bucket_name" -H "Territory: territory_name" -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..."
-```
-
-## Upload an object
-
-| **PUT**  /object |
-| ---------------- |
-
-This interface is used to upload an object, you can write what you want to store directly in the body instead of specifying a file.
-If the upload is successful, you will get the fid of the object. if you want to encrypt the object, you can specify the "Cipher" field in the header of the request and enter a password (the length can not be more than 32 characters), the system will encrypt it automatically.
-
-- Request Header
-
-| key              | description        |
-| ---------------- | ------------------ |
-| Bucket           | bucket name        |
-| Territory        | territory name     |
-| Cipher(optional) | cipher             |
-
-_Identity signature required: yes_
-
-- Request Body
-
-[content]
-
-
-- Request example
-
-```shell
-# curl -X PUT URL/object --data "content" -H "Bucket: bucket_name" -H "Territory: territory_name" -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..."
-```
-
-## Chunked upload
-
-| **PUT**  /chunks |
-| ---------------- |
-
-Compared with uploading the entire file directly, resumable upload has some more parameter requirements, but has the same return result. At the same time, the uploaded file can also be encrypted.
-
-- Request Header
-
-| key           | description        |
-| ------------- | ------------------ |
-| Bucket        | stored bucket name |
-| Territory     | territory name     |
-| Cipher(optional)        | your cipher        |
-| FileName      | file name or alias  |
-| BlockNumber   | The number of chunks the file is to be divided into  |
-| BlockIndex    | index of chunk to be uploaded, [0,BlockNumber)  |
-| TotalSize     | the byte size of the file, the sum of the sizes of all chunks  |
-
-_Identity signature required: yes_
-
-- Request Body
-
-| key  | value        |
-| ---- | ------------ |
-| file | file[binary] |
-
-- Request example
-
-```shell
-# curl -X PUT URL/chunks -F 'file=@test-chunk0;type=application/octet-stream' -H "Bucket: bucket_name" -H "Territory: territory_name" -H "Account: cX..." -H "Message: ..." -H "Signature: 0x... -H FileName: test.log -H BlockNumber: 5 -H BlockIndex: 0 -H TotalSize: 1000"
-```
-
-## Download a file
-
-| **GET**  /download/{fid} |
-| ------------------------ |
-
-This interface is used to download a file with a specified fid. If you encrypted the file when you uploaded it, you also need to tell the gateway your cipher to decrypt your file.
-
-- Request Header
-
-| key              | value    |
-| ---------------- | -------- |
-| Cipher(optional) | cipher   |
-
-
-- Request example
-
-```shell
-# curl -X GET -o <save_file> URL/download/<fid>
-```
-
-## Preview a file
-
-| **GET**  /open/{fid} |
-| -------------------- |
-
-This interface is used to preview a file, it has two prerequisites: one is that the file is not encrypted, and the other is that the file format supports preview.
-
-- Request example
-
-Open in browser: URL/open/<fid>
-
-
-## Delete a file
-
-The delete file interface is used for delete a file.
-
-| **DELETE**  /file/{fid} |
-| ----------------------- |
-
-- Request Header
-
-_Identity signature required: yes_
-
-- Request example
-
-```shell
-# curl -X DELETE URL/file/<fid> -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..."
-```
-
-## Delete a bucket
-
-The delete bucket interface is used for delete a bucket, all files in the bucket will also be deleted together.
-
-| **DELETE**  /bucket/{bucket_name} |
-| -------------------------------- |
-
-- Request Header
-
-_Identity signature required: yes_
-
-- Request example
-
-```shell
-# curl -X DELETE URL/bucket/<bucket_name> -H "Account: cX..." -H "Message: ..." -H "Signature: 0x..."
-```
-
-## View bucket info
-
-| **GET**  /bucket |
-| ---------------- |
-
-This interface is used to view bucket information, including the number of stored files and file IDs.
-
-- Request Header
-
-| key     | value       |
-| ------- | ----------- |
-| Account | cX...       |
-| Bucket  | bucket_name |
-
-- Request example
-
-```shell
-# curl -X GET URL/bucket -H "Account: cX..." -H "Bucket: bucket_name"
-```
-
-## View bucket list
-
-| **GET**  /bucket |
-| ---------------- |
-
-- Request Header
-
-| key     | value  |
-| ------- | ------ |
-| Account | cX...  |
-
-This interface is used to view all buckets.
-
-- Request example
-
-```shell
-# curl -X GET URL/bucket -H "Account: cX..."
-```
-
-## View file metadata
-
-| **GET**  /metadata/{fid} |
-| ------------------------ |
-
-This interface is used to view the basic information of a file.
-
-- Request example
-
-```shell
-# curl -X GET URL/metadata/<fid>
-```
-
-## View version
-
-| **GET**  /version |
-| ----------------- |
-
-This interface is used to view the version number of the gateway.
-
-- Request example
-
-```shell
-# curl -X GET URL/version
-```
+Please refer to [API Description](https://doc.cess.network/products/deoss/api_description)
 
 ## License
 

diff --git a/cmd/cmd/run.go b/cmd/cmd/run.go
@@ -234,7 +234,7 @@ func readEnv() (*confile.Config, error) {
 	}
 
 	// storage network port
-	sport, err := strconv.Atoi(os.Getenv("sport"))
+	sport, err := strconv.Atoi(os.Getenv("storage_port"))
 	if err != nil {
 		return nil, errors.Errorf("invalid storage network port: %v", err)
 	}

diff --git a/node/node.go b/node/node.go
@@ -68,6 +68,8 @@ func (n *Node) Run() {
 	n.Engine = gin.Default()
 	config := cors.DefaultConfig()
 	config.AllowAllOrigins = true
+	config.AddAllowHeaders("*")
+	config.AddExposeHeaders("*")
 	n.Engine.MaxMultipartMemory = MaxMemUsed
 	n.Engine.Use(cors.New(config))
 

diff --git a/node/open_file.go b/node/open_file.go
@@ -137,7 +137,11 @@ func (n *Node) Preview_file(c *gin.Context) {
 	fpath, err = n.retrieve_file(fid, n.fileDir, "")
 	if err != nil {
 		n.Logopen("err", fmt.Sprintf("[%s] Download file [%s] : %v", clientIp, fid, err))
-		c.JSON(http.StatusInternalServerError, "File download failed, please try again later.")
+		if strings.Contains(err.Error(), "being retrieved") {
+			c.JSON(http.StatusForbidden, err.Error())
+			return
+		}
+		c.JSON(http.StatusInternalServerError, "File download failed, it is recommended to use another gateway.")
 		return
 	}
 	n.Logopen("info", fmt.Sprintf("[%s] Download file [%s] suc", clientIp, fid))

diff --git a/node/put_chunks.go b/node/put_chunks.go
@@ -72,7 +72,12 @@ func init() {
 }
 
 func (n *Node) PutChunksHandle(c *gin.Context) {
+	clientIp := c.Request.Header.Get("X-Forwarded-For")
 	account := c.Request.Header.Get(HTTPHeader_Account)
+	if clientIp == "" {
+		clientIp = c.ClientIP()
+	}
+
 	chunkReqLock.Lock()
 	_, ok := chunkReq[account]
 	chunkReqLock.Unlock()
@@ -89,6 +94,7 @@ func (n *Node) PutChunksHandle(c *gin.Context) {
 		chunkReqLock.Lock()
 		chunkReq[account] = time.Now().Unix()
 		chunkReqLock.Unlock()
+		n.Logchunk("info", clientIp+" chunk upload: "+account)
 	}
 
 	var (
@@ -97,7 +103,6 @@ func (n *Node) PutChunksHandle(c *gin.Context) {
 		chunksInfo ChunksInfo
 	)
 
-	clientIp := c.Request.Header.Get("X-Forwarded-For")
 	bucketName := c.Request.Header.Get(HTTPHeader_Bucket)
 	territoryName := c.Request.Header.Get(HTTPHeader_Territory)
 	cipher := c.Request.Header.Get(HTTPHeader_Cipher)
@@ -115,20 +120,13 @@ func (n *Node) PutChunksHandle(c *gin.Context) {
 	}
 	contentLength := c.Request.ContentLength
 
-	if clientIp == "" {
-		clientIp = c.ClientIP()
-	}
-
 	shuntminers := c.Request.Header.Values(HTTPHeader_Miner)
 	longitudes := c.Request.Header.Values(HTTPHeader_Longitude)
 	latitudes := c.Request.Header.Values(HTTPHeader_Latitude)
-	shuntminerslength := len(shuntminers)
-	if shuntminerslength > 0 {
-		n.Logput("info", fmt.Sprintf("shuntminers: %d, %v", shuntminerslength, shuntminers))
-	}
+
 	points, err := coordinate.ConvertToRange(longitudes, latitudes)
 	if err != nil {
-		n.Logput("err", clientIp+" "+err.Error())
+		n.Logchunk("err", clientIp+" "+err.Error())
 		c.JSON(http.StatusBadRequest, err.Error())
 		return
 	}