Skip to content

Commit

Permalink
feat(scenario2): enable multiple solo nodes
Browse files Browse the repository at this point in the history 
Use `make scenario2-setup BASE_PORT=8005 NUM_SOLOS=3` to
set up solo nodes in `t1/8005` through `t1/8007`.  Then
`make scenario2-run-client BASE_PORT=8005` to run the node
in `t1/8005`.

BASE_PORT defaults to 8000, and NUM_SOLOS defaults to 1.
  • Loading branch information
@michaelfig
michaelfig committed Oct 11, 2019
1 parent 412b0a6 commit c8337f9
Show file tree
Hide file tree
Showing 5 changed files with 138 additions and 45 deletions.
40 changes: 30 additions & 10 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ DO_PUSH_LATEST :=
CHAIN_ID = agoric
INITIAL_TOKENS = 1000agmedallion

NUM_SOLOS = 1
BASE_PORT = 8000

ifneq ("$(wildcard /vagrant)","")
# Within a VM. We need to get to the outside.
INSPECT_ADDRESS = 0.0.0.0
Expand All @@ -27,7 +30,7 @@ scenario0-setup:
ve3/bin/pip install setup-solo/

scenario0-run-client:
AG_SOLO_BASEDIR=t9 ve3/bin/ag-setup-solo
AG_SOLO_BASEDIR=t9 ve3/bin/ag-setup-solo --webhost=127.0.0.1:$(BASE_PORT)

scenario0-run-chain:
@echo 'No local chain needs to run in scenario0'
Expand All @@ -39,28 +42,39 @@ scenario1-run-chain:
AG_SETUP_COSMOS_HOME=t8 setup/ag-setup-cosmos bootstrap

scenario1-run-client:
AG_SOLO_BASEDIR=t7 ve3/bin/ag-setup-solo
AG_SOLO_BASEDIR=t7 ve3/bin/ag-setup-solo --webhost=127.0.0.1:$(BASE_PORT)

AGC = ./lib/ag-chain-cosmos
scenario2-setup:
rm -rf ~/.ag-chain-cosmos
rm -f ag-cosmos-chain-state.json
$(AGC) init scenario2-chain --chain-id=$(CHAIN_ID)
rm -rf t1
bin/ag-solo init t1
$(AGC) add-genesis-account `cat t1/ag-cosmos-helper-address` $(INITIAL_TOKENS),100000000stake
echo 'mmmmmmmm' | $(AGC) gentx --home-client=t1/ag-cosmos-helper-statedir --name=ag-solo --amount=1000000stake
mkdir t1
set -e; for port in `seq $(BASE_PORT) $$(($(BASE_PORT) + $(NUM_SOLOS) - 1))`; do \
bin/ag-solo init t1/$$port --webport=$$port; \
case $$port in \
$(BASE_PORT)) toks=$(INITIAL_TOKENS),100000000stake ;; \
*) toks=1agmedallion ;; \
esac; \
$(AGC) add-genesis-account `cat t1/$$port/ag-cosmos-helper-address` $$toks; \
done
echo 'mmmmmmmm' | $(AGC) gentx --home-client=t1/$(BASE_PORT)/ag-cosmos-helper-statedir --name=ag-solo --amount=1000000stake
$(AGC) collect-gentxs
$(AGC) validate-genesis
./setup/set-json.js ~/.ag-chain-cosmos/config/genesis.json --agoric-genesis-overrides
$(MAKE) set-local-gci-ingress
@echo "ROLE=two_chain BOOT_ADDRESS=\`cat t1/ag-cosmos-helper-address\` agc start"
@echo "(cd t1 && ../bin/ag-solo start --role=two_client)"
@echo "ROLE=two_chain BOOT_ADDRESS=\`cat t1/$(BASE_PORT)/ag-cosmos-helper-address\` agc start"
@echo "(cd t1/$(BASE_PORT) && ../bin/ag-solo start --role=two_client)"

scenario2-run-chain:
ROLE=two_chain BOOT_ADDRESS=`cat t1/ag-cosmos-helper-address` $(NODE_DEBUG) `$(BREAK_CHAIN) && echo --inspect-brk` $(AGC) start
set -e; ba=; for acha in t1/*/ag-cosmos-helper-address; do \
ba="$$ba "`cat $$acha`; \
done; \
ROLE=two_chain BOOT_ADDRESS="$$ba" $(NODE_DEBUG) \
`$(BREAK_CHAIN) && echo --inspect-brk` $(AGC) start
scenario2-run-client:
cd t1 && ../bin/ag-solo start --role=two_client
cd t1/$(BASE_PORT) && ../../bin/ag-solo start --role=two_client

scenario3-setup:
rm -rf t3
Expand Down Expand Up @@ -151,7 +165,13 @@ show-local-gci:
@./calc-gci.js ~/.ag-cosmos-chain/config/genesis.json

set-local-gci-ingress:
cd t1 && ../bin/ag-solo set-gci-ingress --chainID=$(CHAIN_ID) `../calc-gci.js ~/.ag-chain-cosmos/config/genesis.json` `../calc-rpcport.js ~/.ag-chain-cosmos/config/config.toml`
set -e; \
gci=`./calc-gci.js ~/.ag-chain-cosmos/config/genesis.json`; \
rpcport=`./calc-rpcport.js ~/.ag-chain-cosmos/config/config.toml`; \
for dir in t1/*; do \
(cd $$dir && \
../../bin/ag-solo set-gci-ingress --chainID=$(CHAIN_ID) $$gci $$rpcport); \
done

start-ag-solo-connected-to-local:
rm -rf t1
Expand Down
9 changes: 6 additions & 3 deletions lib/ag-chain-cosmos
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ let sPort;

function toSwingSet(action, replier) {
console.log(`toSwingSet`, action, replier);
return _toSwingSet(action, replier)
return toSwingSet0(action, replier)
.then(ret => {
console.log(`toSwingSet returning:`, ret);
return ret;
Expand All @@ -60,7 +60,7 @@ function toSwingSet(action, replier) {
});
}

async function _toSwingSet(action, replier) {
async function toSwingSet0(action, _replier) {
if (action.type === 'AG_COSMOS_INIT') {
return true;
}
Expand Down Expand Up @@ -109,7 +109,10 @@ async function _toSwingSet(action, replier) {
};

const vatsdir = path.resolve(__dirname, '../lib/ag-solo/vats');
const argv = [`--role=${ROLE}`, bootAddress];
const argv = [`--role=${ROLE}`];
if (bootAddress) {
argv.push(...bootAddress.trim().split(/\s+/));
}
const s = await launch(mailboxStorage, stateFile, vatsdir, argv);
deliverInbound = s.deliverInbound;
}
Expand Down
21 changes: 16 additions & 5 deletions lib/ag-solo/vats/bootstrap.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,22 @@ function parseArgs(argv) {
const ROLES = {};
let gotRoles = false;
let bootAddress;
const additionalAddresses = [];
argv.forEach(arg => {
const match = arg.match(/^--role=(.*)$/);
if (match) {
ROLES[match[1]] = true;
gotRoles = true;
} else if (!bootAddress && !arg.match(/^-/)) {
bootAddress = arg;
} else if (!arg.match(/^-/)) {
additionalAddresses.push(arg);
}
});
if (!gotRoles) {
ROLES.three_client = true;
}
return [ROLES, bootAddress];
return [ROLES, bootAddress, additionalAddresses];
}

export default function setup(syscall, state, helpers) {
Expand All @@ -48,8 +51,8 @@ export default function setup(syscall, state, helpers) {
return harden({
async bootstrap(argv, vats, devices) {
console.log(`bootstrap(${argv.join(' ')}) called`);
const [ROLES, bootAddress] = parseArgs(argv);
console.log(`Have ROLES`, ROLES, bootAddress);
const [ROLES, bootAddress, additionalAddresses] = parseArgs(argv);
console.log(`Have ROLES`, ROLES, bootAddress, additionalAddresses);

async function addRemote(addr) {
const { transmitter, setReceiver } = await E(vats.vattp).addRemote(addr);
Expand All @@ -59,7 +62,11 @@ export default function setup(syscall, state, helpers) {
D(devices.mailbox).registerInboundHandler(vats.vattp);
await E(vats.vattp).registerMailboxDevice(devices.mailbox);
if (bootAddress) {
await addRemote(bootAddress);
await Promise.all(
[bootAddress, ...additionalAddresses].map(addr =>
addRemote(addr),
),
);
}

if (Object.getOwnPropertyNames(ROLES).length !== 1) {
Expand Down Expand Up @@ -153,7 +160,11 @@ export default function setup(syscall, state, helpers) {
return await bundler.createDemoBundle(nickname);
},
});
await E(vats.comms).addEgress(bootAddress, SC2_INDEX, demoProvider);
await Promise.all(
[bootAddress, ...additionalAddresses].map(addr =>
E(vats.comms).addEgress(addr, SC2_INDEX, demoProvider),
),
);
console.log(`localchain vats initialized`);
return;
}
Expand Down
29 changes: 22 additions & 7 deletions lib/ag-solo/vats/vat-http.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import harden from '@agoric/harden';

import { getReplHandler } from './repl';
// import { getCapTP } from './captp';

// This vat contains the HTTP request handler.

Expand Down Expand Up @@ -32,7 +33,20 @@ function build(E, D) {
getReplHandler(E, homeObjects, msg =>
D(commandDevice).sendBroadcast(msg),
),
// getCapTP(homeObjects),
);
handler.uploadContract = async obj => {
const { connectionID, source, moduleType } = obj;
if (!homeObjects.contractHost) {
throw Error('home.contractHost is not available');
}
console.log('would upload', moduleType);
D(commandDevice).sendBroadcast({
type: 'contractUploaded',
connectionID,
code: 99,
});
};
}
if (ROLES.controller) {
handler.pleaseProvision = obj => {
Expand Down Expand Up @@ -76,13 +90,14 @@ function build(E, D) {

// devices.command invokes our inbound() because we passed to
// registerInboundHandler()
inbound(count, obj) {
console.log(`vat-http.inbound (from browser) ${count}`, obj);
const p = Promise.resolve(handler[obj.type](obj));
p.then(
res => D(commandDevice).sendResponse(count, false, harden(res)),
rej => D(commandDevice).sendResponse(count, true, harden(rej)),
);
async inbound(count, obj) {
try {
console.log(`vat-http.inbound (from browser) ${count}`, obj);
const res = await handler[obj.type](obj);
D(commandDevice).sendResponse(count, false, res);
} catch (rej) {
D(commandDevice).sendResponse(count, true, harden(rej));
}
},
};
}
Expand Down
84 changes: 64 additions & 20 deletions lib/ag-solo/web.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
// Start a network service
const fs = require('fs');
const path = require('path');
const express = require('express');
const WebSocket = require('ws');
const morgan = require('morgan');

const connections = new Set();
const connections = new Map();

const send = (ws, msg) => {
if (ws.readyState === ws.OPEN) {
ws.send(msg);
}
};

export function makeHTTPListener(basedir, port, host, inboundCommand) {
const app = express();
Expand Down Expand Up @@ -34,39 +39,78 @@ export function makeHTTPListener(basedir, port, host, inboundCommand) {
);
});

// accept WebSocket connections at the root path. TODO: I'm guessing that
// this senses the Connection-Upgrade header to distinguish between plain
// accept WebSocket connections at the root path.
// This senses the Connection-Upgrade header to distinguish between plain
// GETs (which should return index.html) and WebSocket requests.. it might
// be better to move the WebSocket listener off to e.g. /ws with a 'path:
// "ws"' option.
const wss = new WebSocket.Server({ server });
let lastConnectionID = 0;

function newConnection(ws, req) {
console.log(`new ws connection`);
connections.add(ws);
lastConnectionID += 1;
const connectionID = lastConnectionID;
const id = `${req.socket.remoteAddress}:${req.socket.remotePort}[${connectionID}]:`;

console.log(id, `new ws connection ${req.url}`);
connections.set(connectionID, ws);

ws.on('error', err => {
console.log('client error', err);
console.log(id, 'client error', err);
});

ws.on('close', (code, reason) => {
console.log('client closed');
connections.delete(ws);
ws.on('close', (_code, _reason) => {
console.log(id, 'client closed');
connections.delete(connectionID);
if (req.url === '/captp') {
inboundCommand({ type: 'CTP_CLOSE', connectionID }).catch(_ => {});
}
});

ws.on('message', message => {
// we ignore messages arriving on the websocket port, it is only for
// outbound broadcasts
console.log(`ignoring message on WS port`, String(message));
});
if (req.url === '/captp') {
// This is a point-to-point connection as well as a broadcast.
inboundCommand({ type: 'CTP_OPEN', connectionID }).catch(err => {
console.log(id, `error establishing connection`, err);
});
ws.on('message', async message => {
try {
// some things use inbound messages
const obj = JSON.parse(message);
obj.connectionID = connectionID;
await inboundCommand(obj);
} catch (e) {
console.log(id, 'client error', e);
send(ws, JSON.stringify({ type: 'CTP_ERROR', error: `${e}` }));
}
});
} else {
ws.on('message', async message => {
// we ignore messages arriving on the default websocket port, it is only for
// outbound broadcasts
console.log(id, `ignoring message on WS port`, String(message));
});
}
}
wss.on('connection', newConnection);

function broadcastJSON(obj) {
connections.forEach(c => {
c.send(JSON.stringify(obj));
});
function sendJSON(obj) {
const { connectionID, ...rest } = obj;
if (connectionID) {
// Point-to-point.
const c = connections.get(connectionID);
if (c) {
send(c, JSON.stringify(rest));
} else {
console.log(`[${connectionID}]: connection not found`);
}
} else {
// Broadcast message.
const json = JSON.stringify(rest);
for (const c of connections.values()) {
send(c, json);
}
}
}

return broadcastJSON;
return sendJSON;
}

0 comments on commit c8337f9

Please sign in to comment.