granter

init-grant-coordinator min-cpu=1 max-cpu=3 sql-kv-tokens=2 sql-sql-tokens=1 sql-leaf=2 sql-root=1
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# No more slots.
try-get work=kv
----
kv: tryGet(1) returned false
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

set-has-waiting-requests work=kv v=true
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Since no more KV slots, couldn't get.
try-get work=sql-kv-response
----
sql-kv-response: tryGet(1) returned false
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

set-has-waiting-requests work=sql-kv-response v=true
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Since no more KV slots, couldn't get.
try-get work=sql-leaf-start
----
sql-leaf-start: tryGet(1) returned false
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

set-has-waiting-requests work=sql-leaf-start v=true
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Since no more KV slots, couldn't get.
try-get work=sql-root-start
----
sql-root-start: tryGet(1) returned false
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

set-has-waiting-requests work=sql-root-start v=true
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

return-grant work=kv
----
kv: returnGrant(1)
kv: granted in chain 1, and returning 1
GrantCoordinator:
(chain: id: 1 active: true index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

set-has-waiting-requests work=kv v=false
----
GrantCoordinator:
(chain: id: 1 active: true index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

set-return-value-from-granted work=kv v=0
----
GrantCoordinator:
(chain: id: 1 active: true index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# The grant chain dies out since kv slots are fully used.
continue-grant-chain work=kv
----
kv: continueGrantChain
GrantCoordinator:
(chain: id: 2 active: false index: 1) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Grant to sql-kv-response consumes a token.
return-grant work=kv
----
kv: returnGrant(1)
sql-kv-response: granted in chain 2, and returning 1
GrantCoordinator:
(chain: id: 2 active: true index: 1) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 1
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Grant to sql-kv-response consumes another token. None left.
continue-grant-chain work=sql-kv-response
----
sql-kv-response: continueGrantChain
sql-kv-response: granted in chain 2, and returning 1
GrantCoordinator:
(chain: id: 2 active: true index: 1) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Even though there are still waiting requests for sql-kv-response, no more
# tokens, so the grant chain can continue past it.
continue-grant-chain work=sql-kv-response
----
sql-kv-response: continueGrantChain
sql-leaf-start: granted in chain 2, and returning 1
GrantCoordinator:
(chain: id: 2 active: true index: 3) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 0, total: 1

continue-grant-chain work=sql-leaf-start
----
sql-leaf-start: continueGrantChain
sql-leaf-start: granted in chain 2, and returning 1
GrantCoordinator:
(chain: id: 2 active: true index: 3) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 2, total: 2 sql-root-start: used: 0, total: 1

# Even though there are still waiting requests for sql-leaf-start, no more
# tokens, so the grant chain can continue past it.
continue-grant-chain work=sql-leaf-start
----
sql-leaf-start: continueGrantChain
sql-root-start: granted in chain 2, and returning 1
GrantCoordinator:
(chain: id: 2 active: true index: 4) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 2, total: 2 sql-root-start: used: 1, total: 1

# sql-root-start ran out of tokens. Grant chain dies out.
continue-grant-chain work=sql-root-start
----
sql-root-start: continueGrantChain
GrantCoordinator:
(chain: id: 3 active: false index: 5) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 2, total: 2 sql-root-start: used: 1, total: 1

# Return sql-leaf-start slot. This will cause another grant chain to start
# which will eventually find a free slot to give to sql-leaf-start.
return-grant work=sql-leaf-start
----
sql-leaf-start: returnGrant(1)
sql-leaf-start: granted in chain 3, and returning 1
GrantCoordinator:
(chain: id: 3 active: true index: 3) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 2, total: 2 sql-root-start: used: 1, total: 1

# Return another sql-leaf-start slot. The grant chain is already active and
# not past this WorkKind, so no grant is done.
return-grant work=sql-leaf-start
----
sql-leaf-start: returnGrant(1)
GrantCoordinator:
(chain: id: 3 active: true index: 3) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# The kv slots are fully used after this tryGet, which succeeds.
try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 3 active: true index: 3) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# This tryGet for kv fails and forces termination of the grant chain.
try-get work=kv
----
kv: tryGet(1) returned false
GrantCoordinator:
(chain: id: 4 active: false index: 3) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

set-has-waiting-requests work=kv v=true
----
GrantCoordinator:
(chain: id: 4 active: false index: 3) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# The grant chain cannot continue since it was force terminated, and a new one
# is not able to restart since there are no free kv slots.
continue-grant-chain work=sql-leaf-start
----
sql-leaf-start: continueGrantChain
GrantCoordinator:
(chain: id: 4 active: false index: 3) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# Pretend that the kv work that was waiting is gone.
set-has-waiting-requests work=kv v=false
----
GrantCoordinator:
(chain: id: 4 active: false index: 3) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# Some other kv work takes without permission.
took-without-permission work=kv
----
kv: tookWithoutPermission(1)
GrantCoordinator:
(chain: id: 4 active: false index: 3) kv: used: 2, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# Refill the tokens and increase the kv slots to 2.
cpu-load runnable=0 procs=1
----
GrantCoordinator:
(chain: id: 4 active: false index: 1) kv: used: 2, high(moderate)-total: 2(0) moderate-clamp: 0 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# Tokens don't get overfull. And kv slots increased to 3. This causes a grant
# to sql-kv-response and the grant chain is again active.
cpu-load runnable=0 procs=1
----
sql-kv-response: granted in chain 4, and returning 1
GrantCoordinator:
(chain: id: 4 active: true index: 1) kv: used: 2, high(moderate)-total: 3(0) moderate-clamp: 0 sql-kv-response: avail: 1
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# Overload and kv slots decreased. Forces termination of grant chain 4.
cpu-load runnable=2 procs=1
----
GrantCoordinator:
(chain: id: 5 active: false index: 1) kv: used: 2, high(moderate)-total: 2(0) moderate-clamp: -1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# Grant chain 4 terminates.
continue-grant-chain work=sql-kv-response
----
sql-kv-response: continueGrantChain
GrantCoordinator:
(chain: id: 5 active: false index: 1) kv: used: 2, high(moderate)-total: 2(0) moderate-clamp: -1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 1, total: 2 sql-root-start: used: 1, total: 1

# Return a slot for sql-leaf-start. Grant chain cannot start since KV slots
# are full.
return-grant work=sql-leaf-start
----
sql-leaf-start: returnGrant(1)
GrantCoordinator:
(chain: id: 5 active: false index: 1) kv: used: 2, high(moderate)-total: 2(0) moderate-clamp: -1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 1, total: 1

# Underload and kv slots increased. The number of procs=4, so can grant 4 at
# the same time. The first 3 get a chain-id=0, i.e., they are not really
# relevant to continuing the grant chain.
cpu-load runnable=2 procs=4
----
sql-kv-response: granted in chain 0, and returning 1
sql-kv-response: granted in chain 0, and returning 1
sql-leaf-start: granted in chain 0, and returning 1
sql-leaf-start: granted in chain 5, and returning 1
GrantCoordinator:
(chain: id: 5 active: true index: 3) kv: used: 2, high(moderate)-total: 3(0) moderate-clamp: 0 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 2, total: 2 sql-root-start: used: 1, total: 1

# There is now a free sql-root-start slot, which the grant chain will get to.
return-grant work=sql-root-start
----
sql-root-start: returnGrant(1)
GrantCoordinator:
(chain: id: 5 active: true index: 3) kv: used: 2, high(moderate)-total: 3(0) moderate-clamp: 0 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 2, total: 2 sql-root-start: used: 0, total: 1

# Continuing with chain-id=0 has no effect.
continue-grant-chain work=sql-kv-response
----
sql-kv-response: continueGrantChain
GrantCoordinator:
(chain: id: 5 active: true index: 3) kv: used: 2, high(moderate)-total: 3(0) moderate-clamp: 0 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 2, total: 2 sql-root-start: used: 0, total: 1

# Continuing chain-id=5 causes a grant to sql-root-start and the chain dies
# out, since do not have a batch of 4 to grant to.
continue-grant-chain work=sql-leaf-start
----
sql-leaf-start: continueGrantChain
sql-root-start: granted in chain 0, and returning 1
GrantCoordinator:
(chain: id: 6 active: false index: 5) kv: used: 2, high(moderate)-total: 3(0) moderate-clamp: 0 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 2, total: 2 sql-root-start: used: 1, total: 1

#####################################################################
# Test skipping of enforcements when CPULoad has high sampling period.
init-grant-coordinator min-cpu=1 max-cpu=3 sql-kv-tokens=1 sql-sql-tokens=1 sql-leaf=2 sql-root=2
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 1
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# No more slots after this slot is granted.
try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 1
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# Since no more KV slots, cannot grant token to sql-kv-response.
try-get work=sql-kv-response
----
sql-kv-response: tryGet(1) returned false
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 1
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# Since no more KV slots, cannot grant token to sql-sql-response.
try-get work=sql-sql-response
----
sql-sql-response: tryGet(1) returned false
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 1, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 1
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# CPULoad shows overload, so cannot increase KV slots, but since it is
# infrequent, slot and token enforcement is disabled.
cpu-load runnable=20 procs=1 infrequent=true
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 1(0) moderate-clamp: -19 sql-kv-response: avail: 1
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# sql-kv-response can get a token.
try-get work=sql-kv-response
----
sql-kv-response: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 1(0) moderate-clamp: -19 sql-kv-response: avail: 0
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# sql-kv-response can get another token, even though tokens are exhausted.
try-get work=sql-kv-response
----
sql-kv-response: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 1(0) moderate-clamp: -19 sql-kv-response: avail: -1
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# sql-sql-response can get a token.
try-get work=sql-sql-response
----
sql-sql-response: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 1(0) moderate-clamp: -19 sql-kv-response: avail: -1
sql-sql-response: avail: 0 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# sql-sql-response can get another token, even though tokens are exhausted.
try-get work=sql-sql-response
----
sql-sql-response: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 1(0) moderate-clamp: -19 sql-kv-response: avail: -1
sql-sql-response: avail: -1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

# KV can get another slot even though slots are exhausted.
try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 1(0) moderate-clamp: -19 sql-kv-response: avail: -1
sql-sql-response: avail: -1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 2

#####################################################################
# Test store grant coordinator.
init-store-grant-coordinator
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 153722867280912930, elastic-disk-bw-tokens-avail: 153722867280912930

# Initial tokens are effectively unlimited.
try-get work=kv v=10000
----
kv-regular: tryGet(10000) returned true
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 153722867280902930, elastic-disk-bw-tokens-avail: 153722867280912930

# Set the io tokens to a smaller value.
set-io-tokens tokens=500
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 500, elastic-disk-bw-tokens-avail: 153722867280912930

# Subtract 100 tokens for elastic work. Note that elastic-disk-bw-tokens-avail also decreases by 100.
took-without-permission work=kv-elastic v=100
----
kv-elastic: tookWithoutPermission(100)
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 400, elastic-disk-bw-tokens-avail: 153722867280912830

# Add 200 tokens.
return-grant work=kv v=200
----
kv-regular: returnGrant(200)
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 600, elastic-disk-bw-tokens-avail: 153722867280912830

# Setup waiting requests that want 400 tokens each.
set-has-waiting-requests work=kv v=true
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 600, elastic-disk-bw-tokens-avail: 153722867280912830

set-return-value-from-granted work=kv v=400
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 600, elastic-disk-bw-tokens-avail: 153722867280912830

# Returning tokens triggers granting and 2 requests will be granted until the
# tokens become <= 0.
return-grant work=kv v=100
----
kv-regular: returnGrant(100)
kv-regular: granted in chain 0, and returning 400
kv-regular: granted in chain 0, and returning 400
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -100, elastic-disk-bw-tokens-avail: 153722867280912830

set-return-value-from-granted work=kv v=100
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -100, elastic-disk-bw-tokens-avail: 153722867280912830

# No tokens to give.
try-get work=kv
----
kv-regular: tryGet(1) returned false
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -100, elastic-disk-bw-tokens-avail: 153722867280912830

# Increment by 50 tokens.
set-io-tokens tokens=50
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -50, elastic-disk-bw-tokens-avail: 153722867280912830

# Return another 50 tokens. Since the number of tokens is 0, there is no
# grant.
return-grant work=kv v=50
----
kv-regular: returnGrant(50)
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 0, elastic-disk-bw-tokens-avail: 153722867280912830

# As soon as the tokens > 0, it will grant.
return-grant work=kv v=1
----
kv-regular: returnGrant(1)
kv-regular: granted in chain 0, and returning 100
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -99, elastic-disk-bw-tokens-avail: 153722867280912830

# Have waiting requests for kv-elastic too.
set-has-waiting-requests work=kv-elastic v=true
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -99, elastic-disk-bw-tokens-avail: 153722867280912830

# The kv-elastic waiting requests need 200 tokens each.
set-return-value-from-granted work=kv-elastic v=200
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -99, elastic-disk-bw-tokens-avail: 153722867280912830

# Since there are regular requests waiting, those are granted first.
return-grant work=kv-elastic v=400
----
kv-elastic: returnGrant(400)
kv-regular: granted in chain 0, and returning 100
kv-regular: granted in chain 0, and returning 100
kv-regular: granted in chain 0, and returning 100
kv-regular: granted in chain 0, and returning 100
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -99, elastic-disk-bw-tokens-avail: 153722867280913230

# No more regular requests waiting.
set-has-waiting-requests work=kv v=false
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -99, elastic-disk-bw-tokens-avail: 153722867280913230

# kv-elastic is granted.
set-io-tokens tokens=100
----
kv-elastic: granted in chain 0, and returning 200
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -199, elastic-disk-bw-tokens-avail: 153722867280913030

set-elastic-disk-bw-tokens tokens=50
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -199, elastic-disk-bw-tokens-avail: 50

# Both kinds of tokens are decremented and become negative.
set-io-tokens tokens=200
----
kv-elastic: granted in chain 0, and returning 200
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: -199, elastic-disk-bw-tokens-avail: -150

# IO tokens become positive. But no grant to elastic work since
# elastic-disk-bw tokens are negative.
set-io-tokens tokens=300
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 101, elastic-disk-bw-tokens-avail: -150

# Regular kv work can get tokens.
try-get work=kv v=10
----
kv-regular: tryGet(10) returned true
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 91, elastic-disk-bw-tokens-avail: -150

# Elastic kv work cannot get tokens.
try-get work=kv-elastic v=10
----
kv-elastic: tryGet(10) returned false
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 91, elastic-disk-bw-tokens-avail: -150

# Still negative.
set-elastic-disk-bw-tokens tokens=50
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 91, elastic-disk-bw-tokens-avail: -100

# Even more IO tokens.
set-io-tokens tokens=400
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 400, elastic-disk-bw-tokens-avail: -100

# Finally both tokens are positive and we grant until the elastic-disk-bw
# tokens become negative.
set-elastic-disk-bw-tokens tokens=120
----
kv-elastic: granted in chain 0, and returning 200
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 200, elastic-disk-bw-tokens-avail: -180

# Models are 0.5x+50, so 0.5*40+50=70. So 70-10=60 additional tokens are
# needed for disk-bw. For io tokens we also apply the ingest model so
# 0.5*0+50=50, which means 110 additional tokens are needed.
store-write-done work=kv-elastic orig-tokens=10 write-bytes=40
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 90, elastic-disk-bw-tokens-avail: -240

store-write-done work=kv orig-tokens=400 write-bytes=40
----
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 370, elastic-disk-bw-tokens-avail: -240

# Both tokens become positive, so one work is granted.
store-write-done work=kv-elastic orig-tokens=400 write-bytes=40
----
kv-elastic: granted in chain 0, and returning 200
GrantCoordinator:
(chain: id: 0 active: false index: 5) io-avail: 450, elastic-disk-bw-tokens-avail: -110


#####################################################################
# Test soft slots
init-grant-coordinator min-cpu=1 max-cpu=6 sql-kv-tokens=2 sql-sql-tokens=1 sql-leaf=2 sql-root=1
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 6 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 6 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 0
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 6 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

return-soft-slots slots=1
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 6 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 6 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=0 procs=4 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 2(2) moderate-clamp: 100 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 2(2) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Over-commitment.
try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 2(2) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Over-commitment.
try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 2(2) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get work=kv
----
kv: tryGet(1) returned false
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 2(2) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# No longer over-committed.
return-soft-slots slots=2
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 2(2) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 0
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 2(2) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=4 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 3(2) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 0
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 3(2) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=1 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 3(3) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 3(3) moderate-clamp: 100 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=3 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 4(3) moderate-clamp: 100 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

return-grant work=kv
----
kv: returnGrant(1)
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 4(3) moderate-clamp: 100 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 4(3) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=2 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 4(4) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=2 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 4(4) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=2
----
requested: 2, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 4(4) moderate-clamp: 100 used-soft: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=2 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 5(5) moderate-clamp: 100 used-soft: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=6 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 5(4) moderate-clamp: 100 used-soft: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=6 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 5(3) moderate-clamp: 100 used-soft: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=6 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 5(3) moderate-clamp: 100 used-soft: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

#####################################################################
# Test soft slots is not higher than regular slots.
init-grant-coordinator min-cpu=1 max-cpu=6 sql-kv-tokens=2 sql-sql-tokens=1 sql-leaf=2 sql-root=1
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 6 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=1
----
requested: 1, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 6 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=2 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 2(2) moderate-clamp: 100 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=1
----
requested: 1, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 2(2) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=2 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 3(3) moderate-clamp: 100 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

return-soft-slots slots=2
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 3(3) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=10 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 3(3) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 1, high(moderate)-total: 3(3) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 2, high(moderate)-total: 3(3) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get work=kv
----
kv: tryGet(1) returned true
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 3, high(moderate)-total: 3(3) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=10 procs=8 clamp=100
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 3, high(moderate)-total: 2(2) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

#####################################################################
# Test clamping down on total moderate slots.
init-grant-coordinator min-cpu=1 max-cpu=100 sql-kv-tokens=2 sql-sql-tokens=1 sql-leaf=2 sql-root=1
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 100 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=1
----
requested: 1, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 100 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=0 procs=8
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 2(2) moderate-clamp: 4 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=1
----
requested: 1, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 2(2) moderate-clamp: 4 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

cpu-load runnable=0 procs=2
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 3(1) moderate-clamp: 1 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# The moderate slots clamp is set to -1, because 10/2-6=-1.
cpu-load runnable=6 procs=10
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 3(0) moderate-clamp: -1 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Making sure total moderate slots is set to 0, and not a negative value.
cpu-load runnable=6 procs=10
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 3(0) moderate-clamp: -1 used-soft: 2 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

#####################################################################
# Make sure the moderate slots clamp has no effect when it is higher than the total moderate slots.
init-grant-coordinator min-cpu=1 max-cpu=3 sql-kv-tokens=2 sql-sql-tokens=1 sql-leaf=2 sql-root=1
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=1
----
requested: 1, granted: 1
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

# Should increase total moderate slots to 2, since the moderate clamp was 3 which is higher.
cpu-load runnable=0 procs=2 clamp=3
----
GrantCoordinator:
(chain: id: 1 active: false index: 5) kv: used: 0, high(moderate)-total: 2(2) moderate-clamp: 3 used-soft: 1 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

init-grant-coordinator min-cpu=1 max-cpu=3 sql-kv-tokens=2 sql-sql-tokens=1 sql-leaf=2 sql-root=1 enabled-soft-slot-granting=false
----
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1

try-get-soft-slots slots=1
----
requested: 1, granted: 0
GrantCoordinator:
(chain: id: 1 active: false index: 0) kv: used: 0, high(moderate)-total: 1(1) moderate-clamp: 3 sql-kv-response: avail: 2
sql-sql-response: avail: 1 sql-leaf-start: used: 0, total: 2 sql-root-start: used: 0, total: 1