Merge branch 'IntersectMBO:master' into master

.github/workflows/haskell.yml

@@ -35,8 +35,13 @@ jobs:
           target-platform: ""
           compiler-nix-name: ${{ matrix.compiler-nix-name }}
           minimal: false
-          # enable IOG flavour to bring in all the crypto libraries we need.
-          iog: true
+          # enable IOG-full flavour to bring in all the crypto libraries we need.
+          iog-full: true
+      - name: "macOS: clean up pre-installed libraries"
+        if: runner.os == 'macOS'
+        # Homebrew's icu4c conflics with Nix's
+        run: |
+          brew uninstall --ignore-dependencies icu4c
       - name: cache cabal
         uses: actions/cache@v3
         with:

.github/workflows/release-ghcr.yml

@@ -34,36 +34,49 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Download image from cache
-        run:
-          nix build --builders "" --max-jobs 0 .#cardano-db-sync-docker
-
       - name: Log in to ghcr.io
         uses: docker/login-action@v2.1.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Upload to ghcr.io
+      - name: Set environment variables
         run: |
           # Downcase the package repository, because docker reference
           # are required to be lower case
           REPO_OWNER="$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')"
-          IMAGE_REF="ghcr.io/$REPO_OWNER/cardano-db-sync"
-          IMAGE_TAG="$GITHUB_REF_NAME"
+          IMAGE_TAG="$(echo $GITHUB_REF_NAME | tr '/' '_')"
+
+          echo "REPO_OWNER=${REPO_OWNER}" >> "$GITHUB_ENV"
+          echo "IMAGE_TAG=${IMAGE_TAG}" >> "$GITHUB_ENV"
+
+      - name: Upload ${{ github.actor }}/cardano-db-sync
+        run: |
+          # Download the image from the nix binary cachhe
+          nix build --builders "" --max-jobs 0 .#cardano-db-sync-docker
+
+          # Push the image
+          skopeo copy \
+            docker-archive:./result \
+            docker://ghcr.io/${REPO_OWNER}/cardano-db-sync:$IMAGE_TAG
+
+          # Also tag it as latest
+          skopeo copy \
+            docker-archive:./result \
+            docker://ghcr.io/${REPO_OWNER}/cardano-db-sync:latest
+
+      - name: Upload ${{ github.actor }}/cardano-smash-server
+        run: |
+          # Download the image from the nix binary cachhe
+          nix build --builders "" --max-jobs 0 .#cardano-smash-server-docker
 
-          # Load the image from the nix build result
-          docker load < result
+          # Push the image
+          skopeo copy \
+            docker-archive:./result \
+            docker://ghcr.io/${REPO_OWNER}/cardano-smash-server:$IMAGE_TAG
 
-          # Tag the image
-          docker image tag \
-            cardano-db-sync:latest  \
-            "${IMAGE_REF}:${IMAGE_TAG}"
           # Also tag it as latest
-          docker image tag \
-            cardano-db-sync:latest  \
-            "${IMAGE_REF}:latest"
-          # Push the tags above
-          docker push "${IMAGE_REF}:${IMAGE_TAG}"
-          docker push "${IMAGE_REF}:latest"
+          skopeo copy \
+            docker-archive:./result \
+            docker://ghcr.io/${REPO_OWNER}/cardano-smash-server:latest

cabal.project

@@ -84,6 +84,7 @@ source-repository-package
   subdir:
     cardano-git-rev
     cardano-node
+    cardano-submit-api
     trace-dispatcher
     trace-forward
     trace-resources

cardano-chain-gen/src/Cardano/Mock/Forging/Tx/Alonzo/ScriptsExamples.hs

@@ -128,8 +128,3 @@ plutusDataEncLen = toShort $ toStrictByteString $ mconcat (encodeListLen 2 : (en
 
 plutusDataEncIndef :: ShortByteString
 plutusDataEncIndef = toShort $ toStrictByteString $ encodeList plutusData2
-
--- toBinaryPlutus :: AlonzoEraScript era => AlonzoScript era -> PlutusBinary
--- toBinaryPlutus as = case mkPlutusScript as of
---   Nothing -> panic "expected Alonzo script"
---   PlutusScript (Plutus _ sbs) -> sbs

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/CommandLineArg/ForceIndex.hs

@@ -19,7 +19,7 @@ checkForceIndexesArg =
   withCustomConfig commandLineForceIndexArgs Nothing babbageConfigDir testLabel $ \_ _ dbSyncEnv -> do
     startDBSync dbSyncEnv
     threadDelay 3_000_000
-    assertEqQuery dbSyncEnv DB.queryPgIndexesCount 162 "there wasn't the correct number of indexes"
+    assertEqQuery dbSyncEnv DB.queryPgIndexesCount 169 "there wasn't the correct number of indexes"
   where
     testLabel = "CLAcheckForceIndexesArg"
     commandLineForceIndexArgs =
@@ -32,7 +32,7 @@ checkNoForceIndexesArg =
   withCustomConfigAndDropDB commandLineNoForceIndexArgs Nothing babbageConfigDir testLabel $ \_ _ dbSyncEnv -> do
     startDBSync dbSyncEnv
     threadDelay 3_000_000
-    assertEqQuery dbSyncEnv DB.queryPgIndexesCount 97 "there wasn't the correct number of indexes"
+    assertEqQuery dbSyncEnv DB.queryPgIndexesCount 104 "there wasn't the correct number of indexes"
   where
     testLabel = "CLAcheckNoForceIndexesArg"
     commandLineNoForceIndexArgs =

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/CommandLineArg/ForceIndex.hs

@@ -17,7 +17,7 @@ checkForceIndexesArg =
     startDBSync dbSync
 
     -- Verify number of DB indexes
-    assertEqQuery dbSync DB.queryPgIndexesCount 162 "unexpected number of indexes"
+    assertEqQuery dbSync DB.queryPgIndexesCount 169 "unexpected number of indexes"
   where
     cliArgs = initCommandLineArgs {claForceIndexes = True}
     testLabel = "conwayCLACheckForceIndexesArg"
@@ -28,7 +28,7 @@ checkNoForceIndexesArg =
     startDBSync dbSync
 
     -- Verify number of DB indexes
-    assertEqQuery dbSync DB.queryPgIndexesCount 97 "unexpected number of indexes"
+    assertEqQuery dbSync DB.queryPgIndexesCount 104 "unexpected number of indexes"
   where
     cliArgs = initCommandLineArgs {claForceIndexes = False}
     testLabel = "conwayCLACheckNoForceIndexesArg"

cardano-db-sync/app/http-get-json-metadata.hs

@@ -1,17 +1,16 @@
 {-# LANGUAGE OverloadedStrings #-}
 
-import Cardano.Db (PoolMetaHash (..), PoolUrl (..))
-import Cardano.DbSync (
-  OffChainFetchError (..),
-  SimplifiedOffChainPoolData (..),
- )
-import Cardano.DbSync.Error (runOrThrowIO)
-import Cardano.DbSync.OffChain.Http (httpGetOffChainPoolData, parseOffChainPoolUrl)
-import Cardano.DbSync.Types (OffChainHashType (..), OffChainUrlType (..))
+import Cardano.Db (PoolMetaHash (..), PoolUrl (..), VoteMetaHash (..), VoteUrl (..))
+import Cardano.DbSync.Error (bsBase16Encode, runOrThrowIO)
+import Cardano.DbSync.OffChain.Http
+import Cardano.DbSync.Types
 import Control.Monad.IO.Class (liftIO)
 import Control.Monad.Trans.Except (ExceptT, runExceptT)
+import qualified Data.ByteString as BS
 import qualified Data.ByteString.Base16 as Base16
-import qualified Data.ByteString.Char8 as BS
+import qualified Data.ByteString.Char8 as BSC
+import Data.ByteString.Lazy (fromStrict)
+import qualified Data.List as List
 import qualified Data.Text as Text
 import qualified Data.Text.IO as Text
 import qualified Network.HTTP.Client as Http
@@ -26,19 +25,36 @@ import System.Console.ANSI.Types (
 import System.Environment (getArgs, getProgName)
 import System.Exit (exitFailure)
 
+{-# ANN module ("HLint: ignore Avoid restricted qualification" :: Text.Text) #-}
+
 main :: IO ()
 main = do
   xs <- getArgs
-  case xs of
-    [url] -> runHttpGet (PoolUrl $ Text.pack url) Nothing
-    [url, hash] -> runHttpGet (PoolUrl $ Text.pack url) (Just $ OffChainPoolHash $ parseHash hash)
+  case cleanOpt xs of
+    [url] -> runGet (isItVote xs) (isItGa xs) (isItLink xs) (Text.pack url) Nothing
+    [url, hash] -> runGet (isItVote xs) (isItGa xs) (isItLink xs) (Text.pack url) (Just $ parseHash hash)
     _otherwise -> usageExit
   where
-    parseHash :: String -> PoolMetaHash
+    parseHash :: String -> BS.ByteString
     parseHash str =
-      case Base16.decode $ BS.pack str of
+      case Base16.decode $ BSC.pack str of
         Left err -> error $ "Failed to Base16 decode hash: " ++ err
-        Right bs -> PoolMetaHash bs
+        Right bs -> bs
+
+    cleanOpt :: [String] -> [String]
+    cleanOpt ls = List.delete "ga" $ List.delete "vote" $ List.delete "pool" ls
+
+    isItVote ls = List.elem "vote" ls || isItGa ls
+    isItGa = List.elem "ga"
+    isItLink = List.elem "url"
+
+    runGet isVote isGa isLink url mhsh
+      | isVote && isLink =
+          runHttpGetVote (VoteUrl url) (VoteMetaHash <$> mhsh) isGa
+      | isVote && not isLink =
+          runGetVote url (VoteMetaHash <$> mhsh) isGa
+      | otherwise =
+          runHttpGetPool (PoolUrl url) (PoolMetaHash <$> mhsh)
 
 -- -------------------------------------------------------------------------------------------------
 
@@ -48,35 +64,65 @@ usageExit = do
   mapM_
     putStrLn
     [ "\nUsage:"
-    , "    " ++ name ++ "<pool metadata utl>"
-    , "    " ++ name ++ "<pool metadata utl> <metadata hash in hex>"
-    , ""
-    , "A debug/test program to debug the pool offchain metadata fetch mechanism.\n"
+    , "    " ++ name ++ "<metadata url>"
+    , "    " ++ name ++ "<metadata url> <metadata hash in hex>"
+    , "with options [vote], [pool], [ga]"
+    , "A debug/test program to debug the offchain metadata fetch mechanism.\n"
     ]
   exitFailure
 
 -- -------------------------------------------------------------------------------------------------
 
-runHttpGet :: PoolUrl -> Maybe OffChainHashType -> IO ()
-runHttpGet poolUrl mHash =
+runHttpGetPool :: PoolUrl -> Maybe PoolMetaHash -> IO ()
+runHttpGetPool poolUrl mHash =
   reportSuccess =<< runOrThrowIO (runExceptT httpGet)
   where
     httpGet :: ExceptT OffChainFetchError IO SimplifiedOffChainPoolData
     httpGet = do
-      request <- parseOffChainPoolUrl poolUrl
+      request <- parseOffChainUrl $ OffChainPoolUrl poolUrl
       manager <- liftIO $ Http.newManager tlsManagerSettings
-      httpGetOffChainPoolData manager request (OffChainPoolUrl poolUrl) mHash
+      httpGetOffChainPoolData manager request poolUrl mHash
 
     reportSuccess :: SimplifiedOffChainPoolData -> IO ()
     reportSuccess spod = do
       case spodContentType spod of
         Nothing -> putStrLn $ orangeText "Warning: No HTTP content-type returned in HTTP response (this should be fixed)."
         Just ct ->
-          if "application/json" `BS.isInfixOf` ct
+          if "application/json" `BSC.isInfixOf` ct
             then putStrLn $ greenText "Success"
-            else putStrLn $ orangeText ("Warning: This should be 'application/json'\nContent-type: " ++ BS.unpack ct)
+            else putStrLn $ orangeText ("Warning: This should be 'application/json'\nContent-type: " ++ BSC.unpack ct)
       Text.putStrLn $ spodJson spod
 
+runHttpGetVote :: VoteUrl -> Maybe VoteMetaHash -> Bool -> IO ()
+runHttpGetVote voteUrl mHash isGa =
+  reportSuccess =<< runOrThrowIO (runExceptT httpGet)
+  where
+    httpGet :: ExceptT OffChainFetchError IO SimplifiedOffChainVoteData
+    httpGet = do
+      request <- parseOffChainUrl $ OffChainVoteUrl voteUrl
+      manager <- liftIO $ Http.newManager tlsManagerSettings
+      httpGetOffChainVoteData manager request voteUrl mHash isGa
+
+    reportSuccess :: SimplifiedOffChainVoteData -> IO ()
+    reportSuccess spod = do
+      case sovaContentType spod of
+        Nothing -> putStrLn $ orangeText "Warning: No HTTP content-type returned in HTTP response (this should be fixed)."
+        Just ct ->
+          if "application/json" `BSC.isInfixOf` ct
+            then putStrLn $ greenText "Success"
+            else putStrLn $ orangeText ("Warning: This should be 'application/json'\nContent-type: " ++ BSC.unpack ct)
+      Text.putStrLn $ sovaJson spod
+
+runGetVote :: Text.Text -> Maybe VoteMetaHash -> Bool -> IO ()
+runGetVote file mExpectedHash isGa = do
+  respBs <- BS.readFile (Text.unpack file)
+  let respLBs = fromStrict respBs
+  (ocvd, val, hsh, mWarning) <- runOrThrowIO $ runExceptT $ parseAndValidateVoteData respBs respLBs mExpectedHash isGa Nothing
+  print ocvd
+  print val
+  print $ bsBase16Encode hsh
+  print mWarning
+
 -- ------------------------------------------------------------------------------------------------
 
 codeGreen :: String

cardano-db-sync/app/test-http-get-json-metadata.hs

@@ -15,11 +15,10 @@ import Cardano.Db (
  )
 import Cardano.DbSync.OffChain.Http (
   httpGetOffChainPoolData,
-  parseOffChainPoolUrl,
+  parseOffChainUrl,
  )
 import Cardano.DbSync.Types (
   OffChainFetchError (..),
-  OffChainHashType (..),
   OffChainUrlType (..),
  )
 import Control.Monad (foldM)
@@ -61,10 +60,10 @@ main = do
     testOne :: Http.Manager -> TestFailure -> TestOffChain -> IO TestFailure
     testOne manager !accum testPoolOffChain = do
       let poolUrl = toUrl testPoolOffChain
-          mHash = Just $ OffChainPoolHash $ toHash testPoolOffChain
+          mHash = Just $ toHash testPoolOffChain
       eres <- runExceptT $ do
-        request <- parseOffChainPoolUrl poolUrl
-        httpGetOffChainPoolData manager request (OffChainPoolUrl poolUrl) mHash
+        request <- parseOffChainUrl (OffChainPoolUrl poolUrl)
+        httpGetOffChainPoolData manager request poolUrl mHash
       case eres of
         Left err -> do
           print err

cardano-db-sync/cardano-db-sync.cabal

@@ -128,6 +128,7 @@ library
                         Cardano.DbSync.OffChain.Http
                         Cardano.DbSync.OffChain.Query
                         Cardano.DbSync.OffChain.Types
+                        Cardano.DbSync.OffChain.Vote.Types
 
                         Cardano.DbSync.LocalStateQuery
                         Cardano.DbSync.StateQuery

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -227,7 +227,7 @@ fullInsertOptions =
 onlyUTxOInsertOptions :: SyncInsertOptions
 onlyUTxOInsertOptions =
   SyncInsertOptions
-    { sioTxOut = TxOutEnable
+    { sioTxOut = TxOutBootstrap (ForceTxIn False)
     , sioLedger = LedgerIgnore
     , sioShelley = ShelleyDisable
     , sioRewards = RewardsConfig True