Skip to content
This repository has been archived by the owner on Mar 20, 2021. It is now read-only.

Commit

Permalink
[PLAT-28297] Add a cli option to specify compression
Browse files Browse the repository at this point in the history
  • Loading branch information
gabrielrussoc committed Feb 10, 2021
1 parent 8aa8e47 commit 5472417
Show file tree
Hide file tree
Showing 8 changed files with 65 additions and 15 deletions.
9 changes: 7 additions & 2 deletions devbox/agent/src/DevboxAgentMain.scala
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import devbox.common._
import devbox.common.Cli
import devbox.common.Cli.Arg
import devbox.common.Cli.pathScoptRead
import devbox.common.CompressionMode
import os.Path

object DevboxAgentMain {
Expand Down Expand Up @@ -190,11 +191,15 @@ object DevboxAgentMain {
}
client.writeMsg(Response.Ack())

case Rpc.WriteChunk(root, path, offset, data) =>
case Rpc.WriteChunk(root, path, offset, data, compressed) =>
val p = wd / root / path

val bytes = compressed match {
case CompressionMode.gzip => Util.gunzip(data.value)
case _ => data.value
}
withWritable(p){
os.write.write(p, data.value, Seq(StandardOpenOption.WRITE), 0, offset)
os.write.write(p, bytes, Seq(StandardOpenOption.WRITE), 0, offset)
}
client.writeMsg(Response.Ack())

Expand Down
12 changes: 12 additions & 0 deletions devbox/common/src/CompressionMode.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package devbox.common

import upickle.default.{ReadWriter, readwriter}

object CompressionMode extends Enumeration {
type CompressionMode = Value
val gzip, ssh, none = Value

implicit val rw: ReadWriter[CompressionMode] = readwriter[Int].bimap[CompressionMode](_.id, apply(_))
}


2 changes: 1 addition & 1 deletion devbox/common/src/Rpc.scala
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ object Rpc{
case class PutLink(root: os.RelPath, path: os.SubPath, dest: String) extends Rpc with PathRpc with Action
object PutLink{ implicit val rw: ReadWriter[PutLink] = macroRW }

case class WriteChunk(root: os.RelPath, path: os.SubPath, offset: Long, data: Bytes) extends Rpc
case class WriteChunk(root: os.RelPath, path: os.SubPath, offset: Long, data: Bytes, compression: CompressionMode.Value) extends Rpc
object WriteChunk{ implicit val rw: ReadWriter[WriteChunk] = macroRW }

case class SetSize(root: os.RelPath, path: os.SubPath, offset: Long) extends Rpc with PathRpc with Action
Expand Down
11 changes: 9 additions & 2 deletions devbox/src/DevboxMain.scala
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ object DevboxMain {
healthCheckInterval: Int = 0,
retryInterval: Int = 0,
noSync: Boolean = false,
proxyGit: Boolean = true)
proxyGit: Boolean = true,
compression: CompressionMode.Value = CompressionMode.ssh)

val signature = Seq(
Arg[Config, String](
Expand Down Expand Up @@ -81,6 +82,11 @@ object DevboxMain {
"Don't sync .git directories and proxy git commands back to the laptop",
(c, v) => c.copy(proxyGit = v)
),
Arg[Config, String](
"compression", None,
s"Enables compression. Possible values are ${CompressionMode.values.mkString(",")}. Defaults to ssh.",
(c, v) => c.copy(compression = CompressionMode.withName(v))
),

)

Expand Down Expand Up @@ -211,7 +217,8 @@ object DevboxMain {
case (path, sig) => sig
}
},
Option(config.syncIgnore)
Option(config.syncIgnore),
config.compression
)

Util.autoclose(syncer){syncer =>
Expand Down
13 changes: 11 additions & 2 deletions devbox/src/syncer/AgentReadWriteActor.scala
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ object AgentReadWriteActor{
case class Close() extends Msg
}
class AgentReadWriteActor(agent: AgentApi,
onResponse: Response => Unit)
onResponse: Response => Unit,
compression: CompressionMode.Value)
(implicit ac: castor.Context,
logger: SyncLogger)
extends castor.StateMachineActor[AgentReadWriteActor.Msg](){
Expand Down Expand Up @@ -210,11 +211,19 @@ class AgentReadWriteActor(agent: AgentApi,
}
}) ()

val data = if (n < byteArr.length) byteArr.take(n) else byteArr
val bytes = new Bytes(
compression match {
case CompressionMode.gzip => Util.gzip(data)
case _ => data
}
)
val msg = Rpc.WriteChunk(
dest,
subPath,
offset,
new Bytes(if (n < byteArr.length) byteArr.take(n) else byteArr)
bytes,
compression,
)
logger.filesAndBytes(0, n)
Some(msg)
Expand Down
5 changes: 4 additions & 1 deletion devbox/src/syncer/Syncer.scala
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,22 @@ class Syncer(agent: AgentApi,
debounceMillis: Int,
proxyGit: Boolean,
signatureTransformer: (os.SubPath, Sig) => Sig,
syncIgnore: Option[String])
syncIgnore: Option[String],
compression: CompressionMode.Value)
(implicit ac: castor.Context, logger: SyncLogger) extends AutoCloseable{


val syncIgnoreRegex = syncIgnore.map(com.google.re2j.Pattern.compile(_))
println(s"Syncing ${mapping.map{case (from, to) => s"$from:$to"}.mkString(", ")}")
println(s"Compression mode: ${compression}")

/** Skippers to use on each repository, used by the SkipScan actor and also sent to the remote endpoint */
val skippers = mapping.map { case (base, _) => Skipper.fromString(ignoreStrategy, base, proxyGit) }

val agentActor: AgentReadWriteActor = new AgentReadWriteActor(
agent,
x => skipActor.send(SkipScanActor.Receive(x)),
compression,
)

val syncActor = new SyncActor(
Expand Down
24 changes: 18 additions & 6 deletions devbox/test/src/devbox/DevboxTests.scala
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ object DevboxTests extends TestSuite{
test("git") - testCase(1, ignoreStrategy = "")
test("restart") - testCase(1, restartInterval = Some(1))
test("reconnect") - testCase(1, randomKillInterval = Some(50))

// Test different compression modes
test("compression-gzip") - testCase(1, compression = CompressionMode.gzip)
// ssh compression here doesn't do anything since we don't ssh anywhere.
test("compression-ssh") - testCase(1, compression = CompressionMode.ssh)
test("compression-none") - testCase(1, compression = CompressionMode.none)
}

test("oslib") {
Expand Down Expand Up @@ -89,7 +95,8 @@ object DevboxTests extends TestSuite{
ignoreStrategy: String = "dotgit",
restartInterval: Option[Int] = None,
randomKillInterval: Option[Int] = None,
parallel: Boolean = true)
parallel: Boolean = true,
compression: CompressionMode.Value = CompressionMode.none)
(implicit tp: TestPath) = {
walkValidate(
tp.value.mkString("-"),
Expand All @@ -99,7 +106,8 @@ object DevboxTests extends TestSuite{
ignoreStrategy = ignoreStrategy,
restartInterval = restartInterval,
randomKillInterval = randomKillInterval,
parallel = parallel
parallel = parallel,
compression = compression,
)
}
def walkValidate(label: String,
Expand All @@ -110,7 +118,8 @@ object DevboxTests extends TestSuite{
ignoreStrategy: String = "dotgit",
restartInterval: Option[Int] = None,
randomKillInterval: Option[Int] = None,
parallel: Boolean = true) = {
parallel: Boolean = true,
compression: CompressionMode.Value = CompressionMode.none) = {

val (src, dest, log, commits, commitsIndicesToCheck, repo) =
initializeWalk(label, uri, validateInterval, commitIndicesToCheck0)
Expand All @@ -134,7 +143,8 @@ object DevboxTests extends TestSuite{
ignoreStrategy,
exitOnError = true,
signatureMapping = (_, sig) => sig,
randomKill = randomKillInterval
randomKill = randomKillInterval,
compression = compression,
)

(logger, ac, syncer)
Expand Down Expand Up @@ -262,7 +272,8 @@ object DevboxTests extends TestSuite{
exitOnError: Boolean,
signatureMapping: (os.SubPath, Sig) => Sig,
healthCheckInterval: Int = 0,
randomKill: Option[Int] = None)
randomKill: Option[Int] = None,
compression: CompressionMode.Value = CompressionMode.none)
(implicit ac: castor.Context,
logger: SyncLogger) = {

Expand All @@ -288,7 +299,8 @@ object DevboxTests extends TestSuite{
debounceMillis,
proxyGit = false,
signatureTransformer = signatureMapping,
syncIgnore = None
syncIgnore = None,
compression = compression
)
syncer
}
Expand Down
4 changes: 3 additions & 1 deletion launcher/src/Main.scala
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package launcher
import cmdproxy.ProxyServer
import devbox.DevboxMain
import devbox.common.Cli
import devbox.common.CompressionMode

object Main {
def main(args: Array[String]): Unit = {
Expand Down Expand Up @@ -60,7 +61,8 @@ object Main {
prepResult.exitCode == 0
},
{ (port: Option[Int]) => Seq(
"ssh", "-C",
"ssh",
"-o", s"Compression=${if (config.compression == CompressionMode.ssh) "yes" else "no"}",
"-o", "ExitOnForwardFailure=yes",
"-o", "ServerAliveInterval=4",
"-o", "ServerAliveCountMax=4"
Expand Down

0 comments on commit 5472417

Please sign in to comment.