Skip to content

Commit

Permalink
Add broker and ewallet schema
Browse files Browse the repository at this point in the history
Add tests for num columns
  • Loading branch information
wongjingping committed Mar 26, 2024
1 parent ecc9242 commit e26bddb
Show file tree
Hide file tree
Showing 8 changed files with 1,257 additions and 21 deletions.
228 changes: 228 additions & 0 deletions defog_data/broker/broker.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
{
"table_metadata": {
"sbCustomer": [
{
"data_type": "varchar(20)",
"column_name": "sbCustId",
"column_description": ""
},
{
"data_type": "varchar(100)",
"column_name": "sbCustName",
"column_description": ""
},
{
"data_type": "varchar(100)",
"column_name": "sbCustEmail",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbCustPhone",
"column_description": ""
},
{
"data_type": "varchar(200)",
"column_name": "sbCustAddress1",
"column_description": ""
},
{
"data_type": "varchar(200)",
"column_name": "sbCustAddress2",
"column_description": ""
},
{
"data_type": "varchar(50)",
"column_name": "sbCustCity",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbCustState",
"column_description": ""
},
{
"data_type": "varchar(50)",
"column_name": "sbCustCountry",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbCustPostalCode",
"column_description": ""
},
{
"data_type": "date",
"column_name": "sbCustJoinDate",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbCustStatus",
"column_description": "possible values: active, inactive, suspended, closed"
}
],
"sbTicker": [
{
"data_type": "varchar(20)",
"column_name": "sbTickerId",
"column_description": ""
},
{
"data_type": "varchar(10)",
"column_name": "sbTickerSymbol",
"column_description": ""
},
{
"data_type": "varchar(100)",
"column_name": "sbTickerName",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbTickerType",
"column_description": "possible values: stock, etf, mutualfund"
},
{
"data_type": "varchar(50)",
"column_name": "sbTickerExchange",
"column_description": ""
},
{
"data_type": "varchar(10)",
"column_name": "sbTickerCurrency",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbTickerDb2x",
"column_description": "2 letter exchange code"
},
{
"data_type": "boolean",
"column_name": "sbTickerIsActive",
"column_description": ""
}
],
"sbDailyPrice": [
{
"data_type": "varchar(20)",
"column_name": "sbDpTickerId",
"column_description": ""
},
{
"data_type": "date",
"column_name": "sbDpDate",
"column_description": ""
},
{
"data_type": "numeric(10,2)",
"column_name": "sbDpOpen",
"column_description": ""
},
{
"data_type": "numeric(10,2)",
"column_name": "sbDpHigh",
"column_description": ""
},
{
"data_type": "numeric(10,2)",
"column_name": "sbDpLow",
"column_description": ""
},
{
"data_type": "numeric(10,2)",
"column_name": "sbDpClose",
"column_description": ""
},
{
"data_type": "bigint",
"column_name": "sbDpVolume",
"column_description": ""
},
{
"data_type": "bigint",
"column_name": "sbDpEpochMs",
"column_description": "epoch milliseconds for timestamp"
},
{
"data_type": "varchar(50)",
"column_name": "sbDpSource",
"column_description": ""
}
],
"sbTransaction": [
{
"data_type": "varchar(50)",
"column_name": "sbTxId",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbTxCustId",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbTxTickerId",
"column_description": ""
},
{
"data_type": "timestamp",
"column_name": "sbTxDateTime",
"column_description": ""
},
{
"data_type": "varchar(20)",
"column_name": "sbTxType",
"column_description": "possible values: buy, sell"
},
{
"data_type": "numeric(10,2)",
"column_name": "sbTxShares",
"column_description": ""
},
{
"data_type": "numeric(10,2)",
"column_name": "sbTxPrice",
"column_description": ""
},
{
"data_type": "numeric(10,2)",
"column_name": "sbTxAmount",
"column_description": ""
},
{
"data_type": "varchar(10)",
"column_name": "sbTxCcy",
"column_description": "transaction currency"
},
{
"data_type": "numeric(10,2)",
"column_name": "sbTxTax",
"column_description": ""
},
{
"data_type": "numeric(10,2)",
"column_name": "sbTxCommission",
"column_description": ""
},
{
"data_type": "varchar(10)",
"column_name": "sbTxKpx",
"column_description": "internal code"
},
{
"data_type": "varchar(25)",
"column_name": "sbTxDateStr",
"column_description": "date as string in yyyyMMdd HH:mm:ss format"
},
{
"data_type": "varchar(10)",
"column_name": "sbTxStatus",
"column_description": "possible values: success, fail, pending"
}
]
},
"glossary": "- sbTicker can be joined to sbDailyPrice on sbTickerId\n- sbCustomer can be joined to sbTransaction on sbCustId\n- sbTicker can be joined to sbTransaction on sbTickerId\n- ADV (Average Daily Volume) for a ticker = AVG(sbDpVolume) from sbDailyPrice table for that ticker\n- ATH (All Time High) price for a ticker = MAX(sbDpHigh) from sbDailyPrice table for that ticker\n- ATP (Average Transaction Price) for a customer = SUM(sbTxAmount)/SUM(sbTxShares) from sbTransaction table for that customer\n- NCT (Net Commission Total) = SUM(sbTxCommission) from sbTransaction table"
}
157 changes: 157 additions & 0 deletions defog_data/broker/broker.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@

-- Dimension tables
CREATE TABLE sbCustomer (
sbCustId varchar(20) PRIMARY KEY,
sbCustName varchar(100) NOT NULL,
sbCustEmail varchar(100) NOT NULL,
sbCustPhone varchar(20),
sbCustAddress1 varchar(200),
sbCustAddress2 varchar(200),
sbCustCity varchar(50),
sbCustState varchar(20),
sbCustCountry varchar(50),
sbCustPostalCode varchar(20),
sbCustJoinDate date NOT NULL,
sbCustStatus varchar(20) NOT NULL -- possible values: active, inactive, suspended, closed
);

CREATE TABLE sbTicker (
sbTickerId varchar(20) PRIMARY KEY,
sbTickerSymbol varchar(10) NOT NULL,
sbTickerName varchar(100) NOT NULL,
sbTickerType varchar(20) NOT NULL, -- possible values: stock, etf, mutualfund
sbTickerExchange varchar(50) NOT NULL,
sbTickerCurrency varchar(10) NOT NULL,
sbTickerDb2x varchar(20), -- 2 letter exchange code
sbTickerIsActive boolean NOT NULL
);

-- Fact tables
CREATE TABLE sbDailyPrice (
sbDpTickerId varchar(20) NOT NULL,
sbDpDate date NOT NULL,
sbDpOpen numeric(10,2) NOT NULL,
sbDpHigh numeric(10,2) NOT NULL,
sbDpLow numeric(10,2) NOT NULL,
sbDpClose numeric(10,2) NOT NULL,
sbDpVolume bigint NOT NULL,
sbDpEpochMs bigint NOT NULL, -- epoch milliseconds for timestamp
sbDpSource varchar(50)
);

CREATE TABLE sbTransaction (
sbTxId varchar(50) PRIMARY KEY,
sbTxCustId varchar(20) NOT NULL,
sbTxTickerId varchar(20) NOT NULL,
sbTxDateTime timestamp NOT NULL,
sbTxType varchar(20) NOT NULL, -- possible values: buy, sell
sbTxShares numeric(10,2) NOT NULL,
sbTxPrice numeric(10,2) NOT NULL,
sbTxAmount numeric(10,2) NOT NULL,
sbTxCcy varchar(10), -- transaction currency
sbTxTax numeric(10,2) NOT NULL,
sbTxCommission numeric(10,2) NOT NULL,
sbTxKpx varchar(10), -- internal code
sbTxDateStr varchar(25), -- date as string in yyyyMMdd HH:mm:ss format
sbTxStatus varchar(10) NOT NULL -- possible values: success, fail, pending
);


-- sbCustomer
INSERT INTO sbCustomer (sbCustId, sbCustName, sbCustEmail, sbCustPhone, sbCustAddress1, sbCustCity, sbCustState, sbCustCountry, sbCustPostalCode, sbCustJoinDate, sbCustStatus) VALUES
('C001', 'John Doe', 'john.doe@email.com', '555-123-4567', '123 Main St', 'Anytown', 'CA', 'USA', '90001', '2020-01-01', 'active'),
('C002', 'Jane Smith', 'jane.smith@email.com', '555-987-6543', '456 Oak Rd', 'Someville', 'NY', 'USA', '10002', '2019-03-15', 'active'),
('C003', 'Bob Johnson', 'bob.johnson@email.com', '555-246-8135', '789 Pine Ave', 'Mytown', 'TX', 'USA', '75000', '2022-06-01', 'inactive'),
('C004', 'Samantha Lee', 'samantha.lee@email.com', '555-135-7902', '246 Elm St', 'Yourtown', 'CA', 'USA', '92101', '2018-09-22', 'suspended'),
('C005', 'Michael Chen', 'michael.chen@email.com', '555-864-2319', '159 Cedar Ln', 'Anothertown', 'FL', 'USA', '33101', '2021-02-28', 'active'),
('C006', 'Emily Davis', 'emily.davis@email.com', '555-753-1904', '753 Maple Dr', 'Mytown', 'TX', 'USA', '75000', '2020-07-15', 'active'),
('C007', 'David Kim', 'david.kim@email.com', '555-370-2648', '864 Oak St', 'Anothertown', 'FL', 'USA', '33101', '2022-11-05', 'active'),
('C008', 'Sarah Nguyen', 'sarah.nguyen@email.com', '555-623-7419', '951 Pine Rd', 'Yourtown', 'CA', 'USA', '92101', '2019-04-01', 'closed'),
('C009', 'William Garcia', 'william.garcia@email.com', '555-148-5326', '258 Elm Ave', 'Anytown', 'CA', 'USA', '90001', '2021-08-22', 'active'),
('C010', 'Jessica Hernandez', 'jessica.hernandez@email.com', '555-963-8520', '147 Cedar Blvd', 'Someville', 'NY', 'USA', '10002', '2020-03-10', 'inactive');

-- sbTicker
INSERT INTO sbTicker (sbTickerId, sbTickerSymbol, sbTickerName, sbTickerType, sbTickerExchange, sbTickerCurrency, sbTickerDb2x, sbTickerIsActive) VALUES
('T001', 'AAPL', 'Apple Inc.', 'stock', 'NASDAQ', 'USD', 'NQ', true),
('T002', 'MSFT', 'Microsoft Corporation', 'stock', 'NASDAQ', 'USD', 'NQ', true),
('T003', 'AMZN', 'Amazon.com, Inc.', 'stock', 'NASDAQ', 'USD', 'NQ', true),
('T004', 'TSLA', 'Tesla, Inc.', 'stock', 'NASDAQ', 'USD', 'NQ', true),
('T005', 'GOOGL', 'Alphabet Inc.', 'stock', 'NASDAQ', 'USD', 'NQ', true),
('T006', 'FB', 'Meta Platforms, Inc.', 'stock', 'NASDAQ', 'USD', 'NQ', true),
('T007', 'BRK.B', 'Berkshire Hathaway Inc.', 'stock', 'NYSE', 'USD', 'NY', true),
('T008', 'JPM', 'JPMorgan Chase & Co.', 'stock', 'NYSE', 'USD', 'NY', true),
('T009', 'V', 'Visa Inc.', 'stock', 'NYSE', 'USD', 'NY', true),
('T010', 'PG', 'Procter & Gamble Company', 'stock', 'NYSE', 'USD', 'NY', true),
('T011', 'SPY', 'SPDR S&P 500 ETF Trust', 'etf', 'NYSE Arca', 'USD', 'NX', true),
('T012', 'QQQ', 'Invesco QQQ Trust', 'etf', 'NASDAQ', 'USD', 'NQ', true),
('T013', 'VTI', 'Vanguard Total Stock Market ETF', 'etf', 'NYSE Arca', 'USD', 'NX', true),
('T014', 'VXUS', 'Vanguard Total International Stock ETF', 'etf', 'NASDAQ', 'USD', 'NQ', true),
('T015', 'VFINX', 'Vanguard 500 Index Fund', 'mutualfund', 'Vanguard', 'USD', 'VG', true),
('T016', 'VTSAX', 'Vanguard Total Stock Market Index Fund', 'mutualfund', 'Vanguard', 'USD', 'VG', true),
('T017', 'VIGAX', 'Vanguard Growth Index Fund', 'mutualfund', 'Vanguard', 'USD', 'VG', true);

-- sbDailyPrice
INSERT INTO sbDailyPrice (sbDpTickerId, sbDpDate, sbDpOpen, sbDpHigh, sbDpLow, sbDpClose, sbDpVolume, sbDpEpochMs, sbDpSource) VALUES
('T001', '2023-04-01', 150.00, 152.50, 148.75, 151.25, 75000000, 1680336000000, 'NYSE'),
('T002', '2023-04-01', 280.00, 282.75, 279.50, 281.00, 35000000, 1680336000000, 'NASDAQ'),
('T003', '2023-04-01', 3200.00, 3225.00, 3180.00, 3210.00, 4000000, 1680336000000, 'NASDAQ'),
('T004', '2023-04-01', 180.00, 185.00, 178.50, 184.25, 20000000, 1680336000000, 'NASDAQ'),
('T005', '2023-04-01', 2500.00, 2525.00, 2475.00, 2510.00, 1500000, 1680336000000, 'NASDAQ'),
('T006', '2023-04-01', 200.00, 205.00, 198.00, 202.50, 15000000, 1680336000000, 'NASDAQ'),
('T007', '2023-04-01', 400000.00, 402500.00, 398000.00, 401000.00, 10000, 1680336000000, 'NYSE'),
('T008', '2023-04-01', 130.00, 132.50, 128.75, 131.00, 12000000, 1680336000000, 'NYSE'),
('T009', '2023-04-01', 220.00, 222.50, 218.00, 221.00, 8000000, 1680336000000, 'NYSE'),
('T010', '2023-04-01', 140.00, 142.00, 139.00, 141.50, 6000000, 1680336000000, 'NYSE'),
('T001', '2023-04-02', 151.50, 153.00, 150.00, 152.00, 70000000, 1680422400000, 'NYSE'),
('T002', '2023-04-02', 281.25, 283.50, 280.00, 282.75, 32000000, 1680422400000, 'NASDAQ'),
('T003', '2023-04-02', 3212.00, 3230.00, 3200.00, 3225.00, 3800000, 1680422400000, 'NASDAQ'),
('T004', '2023-04-02', 184.50, 187.00, 183.00, 186.00, 18000000, 1680422400000, 'NASDAQ'),
('T005', '2023-04-02', 2512.00, 2530.00, 2500.00, 2520.00, 1400000, 1680422400000, 'NASDAQ'),
('T006', '2023-04-02', 203.00, 206.50, 201.00, 205.00, 14000000, 1680422400000, 'NASDAQ'),
('T007', '2023-04-02', 401500.00, 403000.00, 400000.00, 402000.00, 9500, 1680422400000, 'NYSE'),
('T008', '2023-04-02', 131.25, 133.00, 130.00, 132.50, 11000000, 1680422400000, 'NYSE'),
('T009', '2023-04-02', 221.50, 223.00, 220.00, 222.00, 7500000, 1680422400000, 'NYSE'),
('T010', '2023-04-02', 141.75, 143.00, 140.50, 142.25, 5500000, 1680422400000, 'NYSE'),
('T001', '2023-04-03', 152.25, 154.00, 151.00, 153.50, 65000000, 1680508800000, 'NYSE'),
('T002', '2023-04-03', 283.00, 285.00, 281.50, 284.00, 30000000, 1680508800000, 'NASDAQ'),
('T003', '2023-04-03', 3227.00, 3240.00, 3220.00, 3235.00, 3600000, 1680508800000, 'NASDAQ'),
('T004', '2023-04-03', 186.25, 188.50, 185.00, 187.75, 16000000, 1680508800000, 'NASDAQ'),
('T005', '2023-04-03', 2522.00, 2540.00, 2515.00, 2535.00, 1300000, 1680508800000, 'NASDAQ'),
('T006', '2023-04-03', 205.50, 208.00, 203.50, 207.00, 13000000, 1680508800000, 'NASDAQ'),
('T007', '2023-04-03', 402500.00, 404000.00, 401000.00, 403500.00, 9000, 1680508800000, 'NYSE'),
('T008', '2023-04-03', 132.75, 134.50, 131.50, 133.75, 10000000, 1680508800000, 'NYSE'),
('T009', '2023-04-03', 222.25, 224.00, 221.00, 223.50, 7000000, 1680508800000, 'NYSE'),
('T010', '2023-04-03', 142.50, 144.00, 141.50, 143.25, 5000000, 1680508800000, 'NYSE');

-- sbTransaction
INSERT INTO sbTransaction (sbTxId, sbTxCustId, sbTxTickerId, sbTxDateTime, sbTxType, sbTxShares, sbTxPrice, sbTxAmount, sbTxCcy, sbTxTax, sbTxCommission, sbTxKpx, sbTxDateStr, sbTxStatus) VALUES
('TX001', 'C001', 'T001', '2023-04-01 09:30:00', 'buy', 100, 150.00, 15000.00, 'USD', 75.00, 10.00, 'KP001', '20230401 09:30:00', 'success'),
('TX002', 'C002', 'T002', '2023-04-01 10:15:00', 'sell', 50, 280.00, 14000.00, 'USD', 70.00, 10.00, 'KP002', '20230401 10:15:00', 'success'),
('TX003', 'C003', 'T003', '2023-04-01 11:00:00', 'buy', 10, 3200.00, 32000.00, 'USD', 160.00, 20.00, 'KP003', '20230401 11:00:00', 'success'),
('TX004', 'C004', 'T004', '2023-04-01 11:45:00', 'sell', 25, 180.00, 4500.00, 'USD', 22.50, 5.00, 'KP004', '20230401 11:45:00', 'success'),
('TX005', 'C005', 'T005', '2023-04-01 12:30:00', 'buy', 5, 2500.00, 12500.00, 'USD', 62.50, 15.00, 'KP005', '20230401 12:30:00', 'success'),
('TX006', 'C006', 'T006', '2023-04-01 13:15:00', 'sell', 75, 200.00, 15000.00, 'USD', 75.00, 10.00, 'KP006', '20230401 13:15:00', 'success'),
('TX007', 'C007', 'T007', '2023-04-01 14:00:00', 'buy', 1, 400000.00, 400000.00, 'USD', 2000.00, 100.00, 'KP007', '20230401 14:00:00', 'success'),
('TX008', 'C008', 'T008', '2023-04-01 14:45:00', 'sell', 100, 130.00, 13000.00, 'USD', 65.00, 10.00, 'KP008', '20230401 14:45:00', 'success'),
('TX009', 'C009', 'T009', '2023-04-01 15:30:00', 'buy', 50, 220.00, 11000.00, 'USD', 55.00, 10.00, 'KP009', '20230401 15:30:00', 'success'),
('TX010', 'C010', 'T010', '2023-04-01 16:15:00', 'sell', 80, 140.00, 11200.00, 'USD', 56.00, 10.00, 'KP010', '20230401 16:15:00', 'success'),
('TX011', 'C001', 'T001', '2023-04-02 09:30:00', 'sell', 50, 151.50, 7575.00, 'USD', 37.88, 5.00, 'KP011', '20230402 09:30:00', 'success'),
('TX012', 'C002', 'T002', '2023-04-02 10:15:00', 'buy', 30, 281.25, 8437.50, 'USD', 42.19, 7.50, 'KP012', '20230402 10:15:00', 'success'),
('TX013', 'C003', 'T003', '2023-04-02 11:00:00', 'sell', 5, 3212.00, 16060.00, 'USD', 80.30, 15.00, 'KP013', '20230402 11:00:00', 'success'),
('TX014', 'C004', 'T004', '2023-04-02 11:45:00', 'buy', 15, 184.50, 2767.50, 'USD', 13.84, 5.00, 'KP014', '20230402 11:45:00', 'success'),
('TX015', 'C005', 'T005', '2023-04-02 12:30:00', 'sell', 2, 2512.00, 5024.00, 'USD', 25.12, 10.00, 'KP015', '20230402 12:30:00', 'success'),
('TX016', 'C006', 'T006', '2023-04-02 13:15:00', 'buy', 50, 203.00, 10150.00, 'USD', 50.75, 10.00, 'KP016', '20230402 13:15:00', 'success'),
('TX017', 'C007', 'T007', '2023-04-02 14:00:00', 'sell', 1, 401500.00, 401500.00, 'USD', 2007.50, 100.00, 'KP017', '20230402 14:00:00', 'success'),
('TX018', 'C008', 'T008', '2023-04-02 14:45:00', 'buy', 75, 131.25, 9843.75, 'USD', 49.22, 7.50, 'KP018', '20230402 14:45:00', 'success'),
('TX019', 'C009', 'T009', '2023-04-02 15:30:00', 'sell', 25, 221.50, 5537.50, 'USD', 27.69, 5.00, 'KP019', '20230402 15:30:00', 'success'),
('TX020', 'C010', 'T010', '2023-04-02 16:15:00', 'buy', 60, 141.75, 8505.00, 'USD', 42.53, 7.50, 'KP020', '20230402 16:15:00', 'success'),
('TX021', 'C001', 'T001', '2023-04-03 09:30:00', 'buy', 75, 152.25, 11418.75, 'USD', 57.09, 10.00, 'KP021', '20230403 09:30:00', 'success'),
('TX022', 'C002', 'T002', '2023-04-03 10:15:00', 'sell', 40, 283.00, 11320.00, 'USD', 56.60, 10.00, 'KP022', '20230403 10:15:00', 'success'),
('TX023', 'C003', 'T003', '2023-04-03 11:00:00', 'buy', 8, 3227.00, 25816.00, 'USD', 129.08, 20.00, 'KP023', '20230403 11:00:00', 'success'),
('TX024', 'C004', 'T004', '2023-04-03 11:45:00', 'sell', 20, 186.25, 3725.00, 'USD', 18.63, 5.00, 'KP024', '20230403 11:45:00', 'success'),
('TX025', 'C005', 'T005', '2023-04-03 12:30:00', 'buy', 3, 2522.00, 7566.00, 'USD', 37.83, 15.00, 'KP025', '20230403 12:30:00', 'success'),
('TX026', 'C006', 'T006', '2023-04-03 13:15:00', 'sell', 60, 205.50, 12330.00, 'USD', 61.65, 10.00, 'KP026', '20230403 13:15:00', 'success'),
('TX027', 'C007', 'T007', '2023-04-03 14:00:00', 'buy', 1, 402500.00, 402500.00, 'USD', 2012.50, 100.00, 'KP027', '20230403 14:00:00', 'success'),
('TX028', 'C008', 'T008', '2023-04-03 14:45:00', 'sell', 90, 132.75, 11947.50, 'USD', 59.74, 7.50, 'KP028', '20230403 14:45:00', 'success'),
('TX029', 'C009', 'T009', '2023-04-03 15:30:00', 'buy', 40, 222.25, 8890.00, 'USD', 44.45, 10.00, 'KP029', '20230403 15:30:00', 'success'),
('TX030', 'C010', 'T010', '2023-04-03 16:15:00', 'sell', 70, 142.50, 9975.00, 'USD', 49.88, 10.00, 'KP030', '20230403 16:15:00', 'success');
Loading

0 comments on commit e26bddb

Please sign in to comment.