fix: replace fast-atomic-write with steno (#285)

fixes #284 fast-write-atomic hasn't been updated in 5 years, is CJS, and is slower than steno (updated 2 months ago). Benchmarks for various content-types & libraries (though we only use Uint8Arrays) can be found at https://github.com/SgtPooki/fast-write-atomic#benchmarks However, there may be further room for improvement by moving to [fs.createWriteStream](https://nodejs.org/api/fs.html#fscreatewritestreampath-options) ``` ╰─ ✔ ❯ hyperfine --parameter-list branch 284-chore-replace-fast-write-atomic-with-steno,main --setup "git switch {branch} && npm run reset && npm i && npm run build" --runs 20 -w 1 "npm run test:node" Benchmark 1: npm run test:node (branch = 284-chore-replace-fast-write-atomic-with-steno) Time (mean ± σ): 27.212 s ± 0.832 s [User: 34.810 s, System: 6.051 s] Range (min … max): 25.927 s … 29.324 s 20 runs Benchmark 2: npm run test:node (branch = main) Time (mean ± σ): 42.971 s ± 0.637 s [User: 35.297 s, System: 7.534 s] Range (min … max): 42.178 s … 44.796 s 20 runs Summary npm run test:node (branch = 284-chore-replace-fast-write-atomic-with-steno) ran 1.58 ± 0.05 times faster than npm run test:node (branch = main) ``` --- ### Updated benchmarks of `npm run test` as of 2024-04-19 ``` ╭─    ~/code/work/protocol.ai/ipfs/js-stores    main ?1 ╰─ ✔ ❯ hyperfine --parameter-list branch main,test/not-same-event-loop-concurrency,284-chore-replace-fast-write-atomic-with-steno --setup "git switch {branch} && npm run reset && npm i && npm run build && cd packages/datastore-fs" "npm run test" Benchmark 1: npm run test (branch = main) Time (mean ± σ): 99.415 s ± 2.918 s [User: 69.659 s, System: 23.361 s] Range (min … max): 96.134 s … 105.200 s 10 runs Benchmark 2: npm run test (branch = test/not-same-event-loop-concurrency) Time (mean ± σ): 103.456 s ± 3.186 s [User: 74.442 s, System: 25.261 s] Range (min … max): 98.813 s … 108.429 s 10 runs Benchmark 3: npm run test (branch = 284-chore-replace-fast-write-atomic-with-steno) Time (mean ± σ): 80.308 s ± 2.107 s [User: 74.331 s, System: 22.228 s] Range (min … max): 78.219 s … 84.277 s 10 runs Summary npm run test (branch = 284-chore-replace-fast-write-atomic-with-steno) ran 1.24 ± 0.05 times faster than npm run test (branch = main) 1.29 ± 0.05 times faster than npm run test (branch = test/not-same-event-loop-concurrency) [49m1.944s] ╭─    ~/code/work/protocol.ai/ipfs/js-stores    284-chore-re…c-with-steno ?1 ╰─ ✔ ❯ hyperfine --parameter-list branch main,test/not-same-event-loop-concurrency,284-chore-replace-fast-write-atomic-with-steno --setup "git switch {branch} && npm run reset && npm i && npm run build && cd packages/blockstore-fs" "npm run test" Benchmark 1: npm run test (branch = main) Time (mean ± σ): 98.840 s ± 2.612 s [User: 68.486 s, System: 22.585 s] Range (min … max): 97.005 s … 104.396 s 10 runs Benchmark 2: npm run test (branch = test/not-same-event-loop-concurrency) Time (mean ± σ): 105.307 s ± 2.335 s [User: 72.442 s, System: 24.766 s] Range (min … max): 101.167 s … 109.007 s 10 runs Benchmark 3: npm run test (branch = 284-chore-replace-fast-write-atomic-with-steno) Time (mean ± σ): 77.012 s ± 1.829 s [User: 74.442 s, System: 21.938 s] Range (min … max): 75.258 s … 80.825 s 10 runs Summary npm run test (branch = 284-chore-replace-fast-write-atomic-with-steno) ran 1.28 ± 0.05 times faster than npm run test (branch = main) 1.37 ± 0.04 times faster than npm run test (branch = test/not-same-event-loop-concurrency) ```
ipfs · Aug 9, 2024 · c6df8dd · c6df8dd
1 parent f435854
commit c6df8dd
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 22 deletions.
diff --git a/packages/blockstore-fs/package.json b/packages/blockstore-fs/package.json
@@ -77,13 +77,13 @@
   },
   "dependencies": {
     "blockstore-core": "^4.0.0",
-    "fast-write-atomic": "^0.2.1",
     "interface-blockstore": "^5.0.0",
     "interface-store": "^5.0.0",
     "it-glob": "^2.0.6",
     "it-map": "^3.0.5",
     "it-parallel-batch": "^3.0.4",
-    "multiformats": "^13.0.1"
+    "multiformats": "^13.0.1",
+    "steno": "^4.0.2"
   },
   "devDependencies": {
     "aegir": "^42.2.3",

diff --git a/packages/blockstore-fs/src/index.ts b/packages/blockstore-fs/src/index.ts
@@ -14,33 +14,29 @@
 
 import fs from 'node:fs/promises'
 import path from 'node:path'
-import { promisify } from 'node:util'
 import {
   Errors
 } from 'blockstore-core'
-// @ts-expect-error no types
-import fwa from 'fast-write-atomic'
 import glob from 'it-glob'
 import map from 'it-map'
 import parallelBatch from 'it-parallel-batch'
+import { Writer } from 'steno'
 import { NextToLast, type ShardingStrategy } from './sharding.js'
 import type { Blockstore, Pair } from 'interface-blockstore'
 import type { AwaitIterable } from 'interface-store'
 import type { CID } from 'multiformats/cid'
 
-const writeAtomic = promisify(fwa)
-
 /**
  * Write a file atomically
  */
 async function writeFile (file: string, contents: Uint8Array): Promise<void> {
   try {
-    await writeAtomic(file, contents)
+    const writer = new Writer(file)
+    await writer.write(contents)
   } catch (err: any) {
     if (err.syscall === 'rename' && ['ENOENT', 'EPERM'].includes(err.code)) {
-      // fast-write-atomic writes a file to a temp location before renaming it.
-      // On Windows, if the final file already exists this error is thrown.
-      // No such error is thrown on Linux/Mac
+      // steno writes a file to a temp location before renaming it.
+      // If the final file already exists this error is thrown.
       // Make sure we can read & write to this file
       await fs.access(file, fs.constants.F_OK | fs.constants.W_OK)
 

diff --git a/packages/datastore-fs/package.json b/packages/datastore-fs/package.json
@@ -56,12 +56,12 @@
   },
   "dependencies": {
     "datastore-core": "^9.0.0",
-    "fast-write-atomic": "^0.2.1",
     "interface-datastore": "^8.0.0",
     "interface-store": "^5.0.0",
     "it-glob": "^2.0.6",
     "it-map": "^3.0.5",
-    "it-parallel-batch": "^3.0.4"
+    "it-parallel-batch": "^3.0.4",
+    "steno": "^4.0.2"
   },
   "devDependencies": {
     "@types/mkdirp": "^2.0.0",

diff --git a/packages/datastore-fs/src/index.ts b/packages/datastore-fs/src/index.ts
@@ -14,33 +14,29 @@
 
 import fs from 'node:fs/promises'
 import path from 'node:path'
-import { promisify } from 'util'
 import {
   BaseDatastore, Errors
 } from 'datastore-core'
-// @ts-expect-error no types
-import fwa from 'fast-write-atomic'
 import {
   Key, type KeyQuery, type Pair, type Query
 } from 'interface-datastore'
 import glob from 'it-glob'
 import map from 'it-map'
 import parallel from 'it-parallel-batch'
+import { Writer } from 'steno'
 import type { AwaitIterable } from 'interface-store'
 
-const writeAtomic = promisify(fwa)
-
 /**
  * Write a file atomically
  */
 async function writeFile (path: string, contents: Uint8Array): Promise<void> {
   try {
-    await writeAtomic(path, contents)
+    const writer = new Writer(path)
+    await writer.write(contents)
   } catch (err: any) {
     if (err.syscall === 'rename' && ['ENOENT', 'EPERM'].includes(err.code)) {
-      // fast-write-atomic writes a file to a temp location before renaming it.
-      // On Windows, if the final file already exists this error is thrown.
-      // No such error is thrown on Linux/Mac
+      // steno writes a file to a temp location before renaming it.
+      // If the final file already exists this error is thrown.
       // Make sure we can read & write to this file
       await fs.access(path, fs.constants.F_OK | fs.constants.W_OK)