From 431178815c3a0cbf45a7b316fc5c1aa9ea5df0a0 Mon Sep 17 00:00:00 2001 From: Megh Date: Wed, 25 Aug 2021 20:26:42 -0600 Subject: [PATCH 1/2] Adds support for git sources in cocoapods --- docs/strategies/ios/cocoapods.md | 9 ++- src/Strategy/Cocoapods/PodfileLock.hs | 102 ++++++++++++++++-------- test/Cocoapods/PodfileLockSpec.hs | 108 +++++++++++++------------- test/Cocoapods/testdata/Podfile.lock | 13 +++- 4 files changed, 144 insertions(+), 88 deletions(-) diff --git a/docs/strategies/ios/cocoapods.md b/docs/strategies/ios/cocoapods.md index 9278a5d53..7570e4abd 100644 --- a/docs/strategies/ios/cocoapods.md +++ b/docs/strategies/ios/cocoapods.md @@ -39,4 +39,11 @@ DEPENDENCIES: - one (> 4.4) - three (from `Submodules/subproject/.podspec`) - "five/+zlib (7.0.0)" -``` \ No newline at end of file +``` + +## Limitations + +- Pods sourced from local path are not supported (e.g. `pod 'AFNetworking', :path => '~/Documents/AFNetworking'`). +- Pods sourced from http path are not supported (e.g `pod 'JSONKit', :podspec => 'https://example.com/JSONKit.podspec'`). +- Pods sourced from subversion, mercurial, and bazaar are not supported. +- Plugins in Podfiles are ignored. \ No newline at end of file diff --git a/src/Strategy/Cocoapods/PodfileLock.hs b/src/Strategy/Cocoapods/PodfileLock.hs index 33d42140a..7ea8dc6ee 100644 --- a/src/Strategy/Cocoapods/PodfileLock.hs +++ b/src/Strategy/Cocoapods/PodfileLock.hs @@ -4,8 +4,8 @@ module Strategy.Cocoapods.PodfileLock ( PodLock (..), Dep (..), Pod (..), - Section (..), - toSections, + ExternalSource (..), + ExternalGitSource (..), ) where import Control.Effect.Diagnostics ( @@ -14,9 +14,10 @@ import Control.Effect.Diagnostics ( context, run, ) -import Data.Aeson (FromJSON (parseJSON), (.:)) +import Data.Aeson (FromJSON (parseJSON)) +import Data.Aeson.Types (FromJSONKey) import Data.Char qualified as C -import Data.Foldable (traverse_) +import Data.Foldable (asum, traverse_) import Data.Functor (void) import Data.HashMap.Lazy qualified as HashMap (toList) import Data.Map.Strict (Map) @@ -24,11 +25,13 @@ import Data.Map.Strict qualified as Map import Data.Set (Set) import Data.Text (Text) import Data.Void (Void) +import Data.Yaml ((.:), (.:?)) import Data.Yaml qualified as Yaml -import DepTypes (DepType (PodType), Dependency (..), VerConstraint (CEq)) +import DepTypes (DepType (GitType, PodType), Dependency (..), VerConstraint (CEq)) import Effect.Grapher (LabeledGrapher, direct, edge, label, withLabeling) import Effect.ReadFS (ReadFS, readContentsYaml) import Graphing (Graphing) +import Options.Applicative (Alternative ((<|>))) import Path import Text.Megaparsec (MonadParsec (takeWhileP), Parsec, between, empty, errorBundlePretty, parse, some, takeWhile1P) import Text.Megaparsec.Char (char) @@ -36,7 +39,7 @@ import Text.Megaparsec.Char.Lexer qualified as L analyze' :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> m (Graphing Dependency) analyze' file = do - podfileLock <- toSections <$> readContentsYaml file + podfileLock <- readContentsYaml file context "Building dependency graph" $ pure (buildGraph podfileLock) newtype PodfilePkg = PodfilePkg {pkgName :: Text} @@ -45,31 +48,44 @@ newtype PodfilePkg = PodfilePkg {pkgName :: Text} type PodfileGrapher = LabeledGrapher PodfilePkg PodfileLabel newtype PodfileLabel - = PodfileVersion Text + = PodfileVersion (Maybe Text) deriving (Eq, Ord, Show) -toDependency :: PodfilePkg -> Set PodfileLabel -> Dependency -toDependency pkg = foldr applyLabel start +toDependency :: Map Text ExternalSource -> PodfilePkg -> Set PodfileLabel -> Dependency +toDependency externalSrc pkg = foldr applyLabel start where start :: Dependency start = Dependency - { dependencyType = PodType - , dependencyName = pkgName pkg + { dependencyType = depType + , dependencyName = depName , dependencyVersion = Nothing , dependencyLocations = [] , dependencyEnvironments = [] , dependencyTags = Map.empty } + depType :: DepType + depType = case Map.lookup (pkgName pkg) externalSrc of + Just (ExternalGitType _) -> GitType + _ -> PodType + + depName :: Text + depName = case Map.lookup (pkgName pkg) externalSrc of + Just (ExternalGitType gitSrc) -> urlOf gitSrc + _ -> pkgName pkg + applyLabel :: PodfileLabel -> Dependency -> Dependency - applyLabel (PodfileVersion ver) dep = dep{dependencyVersion = Just (CEq ver)} + applyLabel (PodfileVersion ver) dep = dep{dependencyVersion = CEq <$> ver} -buildGraph :: [Section] -> Graphing Dependency -buildGraph sections = - run . withLabeling toDependency $ +buildGraph :: PodLock -> Graphing Dependency +buildGraph lockContent = + run . withLabeling (toDependency $ lockExternalSources lockContent) $ traverse_ addSection sections where + sections :: [Section] + sections = toSections lockContent + addSection :: Has PodfileGrapher sig m => Section -> m () addSection (DependencySection deps) = traverse_ (direct . PodfilePkg . depName) deps addSection (PodSection pods) = traverse_ addSpec pods @@ -77,10 +93,19 @@ buildGraph sections = addSpec :: Has PodfileGrapher sig m => Pod -> m () addSpec pod = do let pkg = PodfilePkg (podName pod) + let pkgVersion = case Map.lookup (podName pod) (lockExternalSources lockContent) of + Just (ExternalGitType src) -> + asum + [ tagOf src + , commitOf src + , branchOf src + ] + _ -> Just $ podVersion pod + -- add edges between spec and specdeps - traverse_ (edge pkg . PodfilePkg . depName) (podSpecs pod) + traverse_ (edge pkg . PodfilePkg . depName) $ podSpecs pod -- add a label for version - label pkg (PodfileVersion (podVersion pod)) + label pkg $ PodfileVersion pkgVersion type Parser = Parsec Void Text @@ -89,9 +114,23 @@ data Section | DependencySection [Dep] deriving (Eq, Ord, Show) +data ExternalSource + = ExternalGitType ExternalGitSource + | ExternalOtherType + deriving (Show, Eq, Ord) + +data ExternalGitSource = ExternalGitSource + { urlOf :: Text + , tagOf :: Maybe Text + , commitOf :: Maybe Text + , branchOf :: Maybe Text + } + deriving (Eq, Ord, Show) + data PodLock = PodLock { lockPods :: [Pod] , lockDeps :: [Dep] + , lockExternalSources :: Map Text ExternalSource } deriving (Show, Eq, Ord) @@ -104,13 +143,7 @@ toSections lockContent = newtype Dep = Dep { depName :: Text } - deriving (Eq, Ord, Show) - -data SourceDep = SourceDep - { sDepName :: Text - , tags :: Map Text Text - } - deriving (Eq, Ord, Show) + deriving (Eq, Ord, Show, FromJSONKey) data Pod = Pod { podName :: Text @@ -119,12 +152,6 @@ data Pod = Pod } deriving (Eq, Ord, Show) -data Remote = Remote - { remoteLocation :: Text - , remoteDeps :: [Dep] - } - deriving (Eq, Ord, Show) - parseName :: Parser Text parseName = lexeme $ takeWhile1P (Just "dep") (not . C.isSpace) @@ -135,9 +162,10 @@ parseNameAndVersion :: Parser (Text, Text) parseNameAndVersion = (,) <$> parseName <*> parseVersion instance FromJSON PodLock where - parseJSON = Yaml.withObject "Podfile.lock content" $ \obj -> do + parseJSON = Yaml.withObject "Podfile.lock content" $ \obj -> PodLock <$> obj .: "PODS" <*> obj .: "DEPENDENCIES" + <*> obj .: "EXTERNAL SOURCES" instance FromJSON Pod where parseJSON (Yaml.String p) = parserPod p Nothing @@ -153,6 +181,18 @@ instance FromJSON Dep where Right name -> pure $ Dep name parseJSON notSupported = fail $ "Expected string, but received: " <> show notSupported +instance FromJSON ExternalSource where + parseJSON = Yaml.withObject "External source" $ \obj -> + do + ExternalGitType + <$> ( ExternalGitSource + <$> obj .: ":git" + <*> obj .:? ":tag" + <*> obj .:? ":commit" + <*> obj .:? ":branch" + ) + <|> pure ExternalOtherType + parserPod :: Text -> Maybe Yaml.Value -> Yaml.Parser Pod parserPod query deps = case parse parseNameAndVersion "" query of Left pErr -> fail $ show $ errorBundlePretty pErr diff --git a/test/Cocoapods/PodfileLockSpec.hs b/test/Cocoapods/PodfileLockSpec.hs index fa4bc741d..0d5cd9c73 100644 --- a/test/Cocoapods/PodfileLockSpec.hs +++ b/test/Cocoapods/PodfileLockSpec.hs @@ -7,22 +7,23 @@ import Data.Map.Strict qualified as Map import Data.Text (Text) import Data.Yaml (decodeEither') import DepTypes ( - DepType (PodType), + DepType (GitType, PodType), Dependency (..), VerConstraint (CEq), ) import GraphUtil (expectDeps, expectDirect, expectEdges) import Strategy.Cocoapods.PodfileLock ( Dep (Dep), + ExternalGitSource (..), + ExternalSource (..), Pod (Pod), - Section (..), + PodLock (..), buildGraph, - toSections, ) import Test.Hspec qualified as T -depOf :: Text -> Maybe Text -> Dependency -depOf name version = +podDepOf :: Text -> Maybe Text -> Dependency +podDepOf name version = Dependency { dependencyType = PodType , dependencyName = name @@ -32,72 +33,79 @@ depOf name version = , dependencyTags = Map.empty } +gitDepOf :: Text -> Maybe Text -> Dependency +gitDepOf name version = + Dependency + { dependencyType = GitType + , dependencyName = name + , dependencyVersion = CEq <$> version + , dependencyLocations = [] + , dependencyEnvironments = [] + , dependencyTags = Map.empty + } + dependencyOne :: Dependency -dependencyOne = depOf "one" (Just "1.0.0") +dependencyOne = podDepOf "one" (Just "1.0.0") dependencyTwo :: Dependency -dependencyTwo = depOf "two" (Just "2.0.0") +dependencyTwo = podDepOf "two" (Just "2.0.0") dependencyThree :: Dependency -dependencyThree = depOf "three" (Just "3.0.0") +dependencyThree = podDepOf "three" (Just "3.0.0") dependencyFour :: Dependency -dependencyFour = depOf "four" (Just "4.0.0") +dependencyFour = podDepOf "four" (Just "4.0.0") dependencyAbnormalName :: Dependency -dependencyAbnormalName = depOf "ab-normal/+name" (Just "2.0.0") - -dependencyNotSoSafeName :: Dependency -dependencyNotSoSafeName = depOf "not-so-safe-name" (Just "2.0.0") - -dependencyGitSourced :: Dependency -dependencyGitSourced = depOf "some-dep-sourced-from-git" (Just "2.0.0") +dependencyAbnormalName = podDepOf "ab-normal/+name" (Just "2.0.0") dependencyGitTagged :: Dependency -dependencyGitTagged = depOf "depWithTag" (Just "2.0.0") +dependencyGitTagged = gitDepOf "git@github.example.com:ab/cap.git" (Just "v1.2.3") dependencyTwoDepA :: Dependency -dependencyTwoDepA = depOf "two_dep_A" Nothing +dependencyTwoDepA = podDepOf "two_dep_A" Nothing dependencyTwoDepB :: Dependency -dependencyTwoDepB = depOf "two-dep-B" Nothing - -podSection :: Section -podSection = - PodSection - [ Pod "one" "1.0.0" [Dep "two", Dep "three", Dep "ab-normal/+name"] - , Pod "two" "2.0.0" [Dep "two_dep_A", Dep "two-dep-B"] - , Pod "three" "3.0.0" [Dep "four"] - , Pod "ab-normal/+name" "2.0.0" [] - , Pod "four" "4.0.0" [] - , Pod "not-so-safe-name" "2.0.0" [] - , Pod "some-dep-sourced-from-git" "2.0.0" [] - , Pod "depWithTag" "2.0.0" [] - ] - -dependencySection :: Section -dependencySection = - DependencySection - [ Dep "one" - , Dep "three" - , Dep "not-so-safe-name" - , Dep "some-dep-sourced-from-git" - , Dep "depWithTag" +dependencyTwoDepB = podDepOf "two-dep-B" Nothing + +pods :: [Pod] +pods = + [ Pod "one" "1.0.0" [Dep "two", Dep "three", Dep "ab-normal/+name"] + , Pod "two" "2.0.0" [Dep "two_dep_A", Dep "two-dep-B"] + , Pod "three" "3.0.0" [Dep "four"] + , Pod "ab-normal/+name" "2.0.0" [] + , Pod "four" "4.0.0" [] + , Pod "depWithTag" "2.0.0" [] + ] + +dependencies :: [Dep] +dependencies = + [ Dep "one" + , Dep "three" + , Dep "depWithTag" + ] + +externalSources :: Map.Map Text ExternalSource +externalSources = + Map.fromList + [ ("depWithTag", ExternalGitType $ ExternalGitSource "git@github.example.com:ab/cap.git" (Just "v1.2.3") Nothing Nothing) + , ("depWithBranch", ExternalGitType $ ExternalGitSource "git@github.example.com:ab/cap.git" Nothing Nothing $ Just "main") + , ("depWithCommit", ExternalGitType $ ExternalGitSource "git@github.example.com:ab/cap.git" Nothing (Just "9a9a9") Nothing) + , ("ChatSecure-Push-iOS", ExternalOtherType) + , ("ChatSecureCore", ExternalOtherType) ] spec :: T.Spec spec = do T.describe "podfile lock analyzer" $ T.it "produces the expected output" $ do - let graph = buildGraph [podSection, dependencySection] + let graph = buildGraph $ PodLock pods dependencies externalSources expectDeps [ dependencyOne , dependencyTwo , dependencyThree , dependencyAbnormalName , dependencyFour - , dependencyNotSoSafeName - , dependencyGitSourced , dependencyGitTagged , dependencyTwoDepA , dependencyTwoDepB @@ -106,8 +114,6 @@ spec = do expectDirect [ dependencyOne , dependencyThree - , dependencyNotSoSafeName - , dependencyGitSourced , dependencyGitTagged ] graph @@ -122,10 +128,8 @@ spec = do graph podLockFile <- T.runIO (BS.readFile "test/Cocoapods/testdata/Podfile.lock") - T.describe "podfile lock parser" $ do - T.it "parses pod and dependency sections" $ - case toSections <$> decodeEither' podLockFile of + T.describe "Podfile.lock parser" $ do + T.it "should parse content" $ + case decodeEither' podLockFile of Left err -> T.expectationFailure $ "failed to parse: " <> show err - Right result -> do - result `T.shouldContain` [podSection] - result `T.shouldContain` [dependencySection] + Right result -> result `T.shouldBe` PodLock pods dependencies externalSources diff --git a/test/Cocoapods/testdata/Podfile.lock b/test/Cocoapods/testdata/Podfile.lock index 2f799145f..a674d112c 100644 --- a/test/Cocoapods/testdata/Podfile.lock +++ b/test/Cocoapods/testdata/Podfile.lock @@ -10,15 +10,11 @@ PODS: - four (= 2.3.3) - "ab-normal/+name (2.0.0)" - four (4.0.0) - - "not-so-safe-name (2.0.0)" - - some-dep-sourced-from-git (2.0.0) - depWithTag (2.0.0) DEPENDENCIES: - one (> 4.4) - three (from `Submodules/subproject/.podspec`) - - "not-so-safe-name (2.0.0)" - - "some-dep-sourced-from-git (from `git@github.example.com:ab/ios.git`, commit `559e6a`)" - depWithTag (from `git@github.example.com:ab/cap.git`, tag `v1.2.3`) SPEC REPOS: @@ -27,6 +23,15 @@ SPEC REPOS: - Appirater EXTERNAL SOURCES: + depWithTag: + :git: "git@github.example.com:ab/cap.git" + :tag: v1.2.3 + depWithBranch: + :git: "git@github.example.com:ab/cap.git" + :branch: main + depWithCommit: + :git: "git@github.example.com:ab/cap.git" + :commit: 9a9a9 ChatSecure-Push-iOS: :path: Submodules/ChatSecure-Push-iOS/ChatSecure-Push-iOS.podspec ChatSecureCore: From d6a9dc35b14a0b5ef2f5cf4390f977d1dab71ff8 Mon Sep 17 00:00:00 2001 From: Megh Date: Fri, 27 Aug 2021 08:30:59 -0600 Subject: [PATCH 2/2] updates changelog --- Changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Changelog.md b/Changelog.md index afc01ff4c..708f0b5c3 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,9 @@ # Spectrometer Changelog +## v2.15.9 + +- CocoaPods: Supports git sources in `Podfile.lock` analysis. ([#345](https://github.com/fossas/spectrometer/pull/345)) + ## v2.15.8 - `fossa analyze --experimental-enable-monorepo` now turns off proprietary language scanning by default, and has this feature controlled by a feature flag ([#343](https://github.com/fossas/spectrometer/pull/343))