-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #533 from evo-lua/async-file-reader
Add an AsyncFileReader module to the FileSystem API
- Loading branch information
Showing
5 changed files
with
324 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
local AsyncFileReader = require("AsyncFileReader") | ||
|
||
local console = require("console") | ||
local uv = require("uv") | ||
|
||
console.startTimer("Generating test fixtures") | ||
local SAMPLE_SIZE = 250 | ||
local SMALL_FILE_PATH = path.join("Tests", "Fixtures", "test-small.txt") | ||
local LARGE_FILE_PATH = path.join("Tests", "Fixtures", "test-large.txt") | ||
local HUGE_FILE_PATH = path.join("Tests", "Fixtures", "test-huge.txt") | ||
local SMALL_FILE_SIZE_IN_BYTES = math.min(AsyncFileReader.CHUNK_SIZE_IN_BYTES - 1, 32) | ||
local LARGE_FILE_SIZE_IN_BYTES = 4 * AsyncFileReader.CHUNK_SIZE_IN_BYTES + 1 | ||
local HUGE_FILE_SIZE_IN_BYTES = 1024 * 1024 * 32 | ||
local SMALL_FILE_CONTENTS = string.rep("A", SMALL_FILE_SIZE_IN_BYTES) | ||
local LARGE_FILE_CONTENTS = string.rep("A", LARGE_FILE_SIZE_IN_BYTES) | ||
local HUGE_FILE_CONTENTS = string.rep("A", HUGE_FILE_SIZE_IN_BYTES) | ||
C_FileSystem.WriteFile(SMALL_FILE_PATH, SMALL_FILE_CONTENTS) | ||
C_FileSystem.WriteFile(LARGE_FILE_PATH, LARGE_FILE_CONTENTS) | ||
C_FileSystem.WriteFile(HUGE_FILE_PATH, HUGE_FILE_CONTENTS) | ||
console.stopTimer("Generating test fixtures") | ||
|
||
math.randomseed(os.clock()) | ||
local availableBenchmarks = { | ||
function() | ||
local label = "[ASYNC] Loading a small file repeatedly, many times" | ||
console.startTimer(label) | ||
for i = 1, SAMPLE_SIZE, 1 do | ||
AsyncFileReader:LoadFileContents(SMALL_FILE_PATH) | ||
end | ||
uv.run() | ||
console.stopTimer(label) | ||
end, | ||
function() | ||
local label = "[ASYNC] Loading a large file repeatedly, many times" | ||
console.startTimer(label) | ||
for i = 1, SAMPLE_SIZE, 1 do | ||
AsyncFileReader:LoadFileContents(LARGE_FILE_PATH) | ||
end | ||
uv.run() | ||
console.stopTimer(label) | ||
end, | ||
function() | ||
local label = "[ASYNC] Loading a huge file repeatedly, many times" | ||
console.startTimer(label) | ||
for i = 1, SAMPLE_SIZE, 1 do | ||
AsyncFileReader:LoadFileContents(HUGE_FILE_PATH) | ||
end | ||
uv.run() | ||
console.stopTimer(label) | ||
end, | ||
function() | ||
local label = "[SYNC] Loading a small file repeatedly, many times" | ||
console.startTimer(label) | ||
for i = 1, SAMPLE_SIZE, 1 do | ||
C_FileSystem.ReadFile(SMALL_FILE_PATH) | ||
end | ||
console.stopTimer(label) | ||
end, | ||
function() | ||
local label = "[SYNC] Loading a large file repeatedly, many times" | ||
console.startTimer(label) | ||
for i = 1, SAMPLE_SIZE, 1 do | ||
C_FileSystem.ReadFile(LARGE_FILE_PATH) | ||
end | ||
console.stopTimer(label) | ||
end, | ||
function() | ||
local label = "[SYNC] Loading a huge file repeatedly, many times" | ||
console.startTimer(label) | ||
for i = 1, SAMPLE_SIZE, 1 do | ||
C_FileSystem.ReadFile(HUGE_FILE_PATH) | ||
end | ||
console.stopTimer(label) | ||
end, | ||
} | ||
|
||
table.shuffle(availableBenchmarks) | ||
|
||
for _, benchmark in ipairs(availableBenchmarks) do | ||
benchmark() | ||
end | ||
|
||
console.startTimer("Removing test fixtures") | ||
C_FileSystem.Delete(SMALL_FILE_PATH) | ||
C_FileSystem.Delete(LARGE_FILE_PATH) | ||
C_FileSystem.Delete(HUGE_FILE_PATH) | ||
console.stopTimer("Removing test fixtures") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,132 @@ | ||
local etrace = require("etrace") | ||
local uv = require("uv") | ||
|
||
local math_ceil = math.ceil | ||
|
||
local AsyncFileReader = { | ||
events = { | ||
"FILE_REQUEST_STARTED", | ||
"FILE_REQUEST_FAILED", | ||
"FILE_REQUEST_COMPLETED", | ||
"FILE_DESCRIPTOR_OPENED", | ||
"FILE_STATUS_AVAILABLE", | ||
"FILE_CHUNK_AVAILABLE", | ||
"FILE_CONTENTS_AVAILABLE", | ||
"FILE_DESCRIPTOR_CLOSED", | ||
}, | ||
FILE_MODE_READONLY = 292, -- Octal: 444 | ||
CHUNK_SIZE_IN_BYTES = 1024 * 256, | ||
} | ||
|
||
etrace.register(AsyncFileReader.events) | ||
|
||
function AsyncFileReader:LoadFileContents(fileSystemPath) | ||
local payload = { | ||
fileSystemPath = fileSystemPath, | ||
} | ||
|
||
uv.fs_open(fileSystemPath, "r", AsyncFileReader.FILE_MODE_READONLY, function(errorMessage, fileDescriptor) | ||
payload.errorMessage = errorMessage | ||
payload.fileDescriptor = fileDescriptor | ||
|
||
if errorMessage then | ||
EVENT("FILE_REQUEST_FAILED", payload) | ||
return | ||
end | ||
|
||
EVENT("FILE_DESCRIPTOR_OPENED", payload) | ||
end) | ||
|
||
EVENT("FILE_REQUEST_STARTED", payload) | ||
end | ||
|
||
function AsyncFileReader:FILE_DESCRIPTOR_OPENED(event, payload) | ||
uv.fs_fstat(payload.fileDescriptor, function(errorMessage, stat) | ||
payload.errorMessage = errorMessage | ||
payload.stat = stat | ||
|
||
if errorMessage then | ||
EVENT("FILE_REQUEST_FAILED", payload) | ||
return | ||
end | ||
|
||
EVENT("FILE_STATUS_AVAILABLE", payload) | ||
end) | ||
end | ||
|
||
function AsyncFileReader:FILE_STATUS_AVAILABLE(event, payload) | ||
payload.lastChunkIndex = math_ceil(payload.stat.size / AsyncFileReader.CHUNK_SIZE_IN_BYTES) | ||
payload.chunkIndex = 0 | ||
payload.cursorPosition = 0 | ||
|
||
if payload.stat.type == "directory" then | ||
-- On Windows, read requests on directories succeed without returning any data | ||
-- Simulating the error returned on other platforms here allows providing a consistent interface | ||
payload.errorMessage = "EISDIR: illegal operation on a directory" -- Should use uv_strerror but it isn't currently bound | ||
EVENT("FILE_REQUEST_FAILED", payload) | ||
return | ||
end | ||
|
||
self:ReadNextFileChunk(payload) | ||
end | ||
|
||
function AsyncFileReader:ReadNextFileChunk(payload) | ||
-- Consecutive chunked reads may overwrite the payload unless copied | ||
payload = table.scopy(payload) | ||
|
||
local totalFileSizeInBytes = payload.stat.size | ||
local startOffset = payload.cursorPosition | ||
|
||
local numLeftoverBytes = math.min(AsyncFileReader.CHUNK_SIZE_IN_BYTES, totalFileSizeInBytes - startOffset) | ||
if numLeftoverBytes <= 0 then | ||
EVENT("FILE_CONTENTS_AVAILABLE", payload) | ||
return | ||
end | ||
|
||
uv.fs_read(payload.fileDescriptor, numLeftoverBytes, startOffset, function(errorMessage, chunk) | ||
payload.errorMessage = errorMessage | ||
payload.chunk = chunk | ||
|
||
if errorMessage then | ||
EVENT("FILE_REQUEST_FAILED", payload) | ||
return | ||
end | ||
|
||
EVENT("FILE_CHUNK_AVAILABLE", payload) | ||
|
||
local newOffset = startOffset + numLeftoverBytes | ||
payload.cursorPosition = newOffset | ||
if newOffset < totalFileSizeInBytes then | ||
return self:ReadNextFileChunk(payload) | ||
end | ||
|
||
EVENT("FILE_CONTENTS_AVAILABLE", payload) | ||
end) | ||
|
||
payload.chunkIndex = payload.chunkIndex + 1 | ||
end | ||
|
||
function AsyncFileReader:FILE_CONTENTS_AVAILABLE(event, payload) | ||
uv.fs_close(payload.fileDescriptor, function() | ||
EVENT("FILE_DESCRIPTOR_CLOSED", payload) | ||
end) | ||
|
||
EVENT("FILE_REQUEST_COMPLETED", payload) | ||
end | ||
|
||
function AsyncFileReader:FILE_REQUEST_FAILED(event, payload) | ||
if not payload.fileDescriptor then | ||
return | ||
end | ||
|
||
uv.fs_close(payload.fileDescriptor, function() | ||
EVENT("FILE_DESCRIPTOR_CLOSED", payload) | ||
end) | ||
end | ||
|
||
etrace.subscribe("FILE_DESCRIPTOR_OPENED", AsyncFileReader) | ||
etrace.subscribe("FILE_STATUS_AVAILABLE", AsyncFileReader) | ||
etrace.subscribe("FILE_CONTENTS_AVAILABLE", AsyncFileReader) | ||
etrace.subscribe("FILE_REQUEST_FAILED", AsyncFileReader) | ||
|
||
return AsyncFileReader |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
local etrace = require("etrace") | ||
local uv = require("uv") | ||
|
||
local AsyncFileReader = require("AsyncFileReader") | ||
|
||
local OLD_CHUNK_SIZE = AsyncFileReader.CHUNK_SIZE_IN_BYTES | ||
local NEW_CHUNKS_SIZE = 2 -- No point in generating large payloads here | ||
local MAX_LENGTH_CHUNK = string.rep("A", NEW_CHUNKS_SIZE) | ||
AsyncFileReader.CHUNK_SIZE_IN_BYTES = NEW_CHUNKS_SIZE | ||
|
||
local SMALL_TEST_FILE = "temp-small.txt" | ||
local LARGE_TEST_FILE = "temp-large.txt" | ||
local FILE_CONTENTS_SMALL = string.rep("A", NEW_CHUNKS_SIZE - 1) | ||
local FILE_CONTENTS_LARGE = MAX_LENGTH_CHUNK .. MAX_LENGTH_CHUNK .. "A" | ||
C_FileSystem.WriteFile(SMALL_TEST_FILE, FILE_CONTENTS_SMALL) | ||
C_FileSystem.WriteFile(LARGE_TEST_FILE, FILE_CONTENTS_LARGE) | ||
|
||
describe("AsyncFileReader", function() | ||
describe("LoadFileContents", function() | ||
before(function() | ||
etrace.enable(AsyncFileReader.events) | ||
end) | ||
|
||
after(function() | ||
etrace.clear() | ||
etrace.disable(AsyncFileReader.events) | ||
end) | ||
|
||
it("should fail if the given path is invalid", function() | ||
AsyncFileReader:LoadFileContents("does-not-exist") | ||
uv.run() | ||
|
||
local events = etrace.filter("FILE_REQUEST_FAILED") | ||
assertEquals(#events, 1) | ||
|
||
assertEquals(events[1].name, "FILE_REQUEST_FAILED") | ||
assertEquals(events[1].payload.fileSystemPath, "does-not-exist") | ||
assertEquals(events[1].payload.errorMessage, "ENOENT: no such file or directory: does-not-exist") | ||
end) | ||
|
||
it("should fail if the given path refers to a directory", function() | ||
AsyncFileReader:LoadFileContents("Runtime") | ||
uv.run() | ||
|
||
local events = etrace.filter("FILE_REQUEST_FAILED") | ||
assertEquals(#events, 1) | ||
|
||
assertEquals(events[1].name, "FILE_REQUEST_FAILED") | ||
assertEquals(events[1].payload.fileSystemPath, "Runtime") | ||
assertEquals(events[1].payload.errorMessage, "EISDIR: illegal operation on a directory") | ||
end) | ||
|
||
it("should read a single chunk if the file isn't large enough to warrant buffering", function() | ||
AsyncFileReader:LoadFileContents(SMALL_TEST_FILE) | ||
uv.run() | ||
|
||
local events = etrace.filter("FILE_CHUNK_AVAILABLE") | ||
local numExpectedChunks = 1 | ||
assertEquals(#events, numExpectedChunks) | ||
|
||
assertEquals(events[1].name, "FILE_CHUNK_AVAILABLE") | ||
assertEquals(events[1].payload.cursorPosition, 1) | ||
assertEquals(events[1].payload.chunk, "A") | ||
assertEquals(events[1].payload.fileSystemPath, SMALL_TEST_FILE) | ||
assertEquals(events[1].payload.lastChunkIndex, 1) | ||
assertEquals(events[1].payload.chunkIndex, 1) | ||
end) | ||
|
||
it("should read multiple chunks if the file is large enough to warrant buffering", function() | ||
AsyncFileReader:LoadFileContents(LARGE_TEST_FILE) | ||
uv.run() | ||
|
||
local events = etrace.filter("FILE_CHUNK_AVAILABLE") | ||
local numExpectedChunks = 3 | ||
assertEquals(#events, numExpectedChunks) | ||
|
||
assertEquals(events[1].name, "FILE_CHUNK_AVAILABLE") | ||
assertEquals(events[1].payload.cursorPosition, 2) | ||
assertEquals(events[1].payload.chunk, "AA") | ||
assertEquals(events[1].payload.fileSystemPath, LARGE_TEST_FILE) | ||
assertEquals(events[1].payload.lastChunkIndex, 3) | ||
assertEquals(events[1].payload.chunkIndex, 1) | ||
|
||
assertEquals(events[2].name, "FILE_CHUNK_AVAILABLE") | ||
assertEquals(events[2].payload.cursorPosition, 4) | ||
assertEquals(events[2].payload.chunk, "AA") | ||
assertEquals(events[2].payload.fileSystemPath, LARGE_TEST_FILE) | ||
assertEquals(events[2].payload.lastChunkIndex, 3) | ||
assertEquals(events[2].payload.chunkIndex, 2) | ||
|
||
assertEquals(events[3].name, "FILE_CHUNK_AVAILABLE") | ||
assertEquals(events[3].payload.cursorPosition, 5) | ||
assertEquals(events[3].payload.chunk, "A") | ||
assertEquals(events[3].payload.fileSystemPath, LARGE_TEST_FILE) | ||
assertEquals(events[3].payload.lastChunkIndex, 3) | ||
assertEquals(events[3].payload.chunkIndex, 3) | ||
end) | ||
end) | ||
end) | ||
|
||
C_FileSystem.Delete(SMALL_TEST_FILE) | ||
C_FileSystem.Delete(LARGE_TEST_FILE) | ||
AsyncFileReader.CHUNK_SIZE_IN_BYTES = OLD_CHUNK_SIZE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters