From bdc216c4c45778b62cff8a0805e4d69c67f0c9d0 Mon Sep 17 00:00:00 2001 From: Megh Date: Tue, 31 Aug 2021 19:12:28 -0600 Subject: [PATCH 1/9] Adds swift package manager manifest support --- docs/quickreference/swiftpm.md | 12 ++ docs/strategies.md | 2 +- docs/strategies/ios/swift.md | 73 +++++++++ docs/userguide.md | 2 + spectrometer.cabal | 3 + src/App/Fossa/ManualDeps.hs | 1 + src/DepTypes.hs | 2 + src/Srclib/Converter.hs | 1 + src/Strategy/Swift/PackageSwift.hs | 248 +++++++++++++++++++++++++++++ src/Strategy/SwiftPM.hs | 55 +++++++ test/Swift/PackageSwiftSpec.hs | 125 +++++++++++++++ test/Swift/testdata/Package.swift | 64 ++++++++ 12 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 docs/quickreference/swiftpm.md create mode 100644 docs/strategies/ios/swift.md create mode 100644 src/Strategy/Swift/PackageSwift.hs create mode 100644 src/Strategy/SwiftPM.hs create mode 100644 test/Swift/PackageSwiftSpec.hs create mode 100644 test/Swift/testdata/Package.swift diff --git a/docs/quickreference/swiftpm.md b/docs/quickreference/swiftpm.md new file mode 100644 index 000000000..94b9940f4 --- /dev/null +++ b/docs/quickreference/swiftpm.md @@ -0,0 +1,12 @@ +# Quick reference: Swift Package Manager + +## Requirements + +**Minimum** + +- `Package.swift` file present in your project + +## Project discovery + +Directories containing `Package.swift` files are considered +projects managed by swift package manager. diff --git a/docs/strategies.md b/docs/strategies.md index cb83072e4..bd173a659 100644 --- a/docs/strategies.md +++ b/docs/strategies.md @@ -76,7 +76,7 @@ The CLI supports the following strategies: - [golang](strategies/golang.md) (gomodules, dep, glide) - [gradle](strategies/gradle.md) - [haskell](strategies/haskell.md) (cabal, stack) -- [iOS](strategies/ios.md) (Carthage, Cocoapods) +- [iOS](strategies/ios.md) (Carthage, Cocoapods, Swift package manager) - [maven](strategies/maven.md) - [nodejs](strategies/nodejs.md) (yarn, npmcli) - [python](strategies/python.md) (conda, pipenv, setuptools) diff --git a/docs/strategies/ios/swift.md b/docs/strategies/ios/swift.md new file mode 100644 index 000000000..762774620 --- /dev/null +++ b/docs/strategies/ios/swift.md @@ -0,0 +1,73 @@ +# Swift Package Manager + +## Project Discovery + +Find all swift manifest files, named: `Package.swift` + +# Swift Package Analysis + +| Strategy | Direct Deps | Deep Deps | Edges | Classifies Test Dependencies | +| --------------------------------------------- | ------------------ | --------- | ----- | ---------------------------- | +| parse package dependencies in `Package.swift` | :white_check_mark: | :x: | :x: | :x: | + +- Manifest file - `Package.swift`, must begin with `// swift-tools-version:` string, followed by version number specifier. +- We follow swift package manager's convention, and presume properties of package are defined in a single nested initializer statement, and are not modified after initialization. + +## Limitation + +- Path dependencies are ignored in the analyses (e.g. `package(path: "./../local-pkg")`) + +## Example + +Create Package.swift file in the directory. Add dependencies, targets, products, and source code. Example Package.swift file is shown below. By convention, the properties of a Package are defined in a single nested initializer statement, and not modified after initialization. + +```swift +// swift-tools-version:4.0 +import PackageDescription + +let package = Package( + name: "DeckOfPlayingCards", + products: [ + .library(name: "DeckOfPlayingCards", targets: ["DeckOfPlayingCards"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/example-package-fisheryates.git", from: "2.0.0"), + .package(url: "https://github.com/apple/example-package-playingcard.git", from: "3.0.0"), + ], + targets: [ + .target( + name: "DeckOfPlayingCards", + dependencies: ["FisherYates", "PlayingCard"]), + .testTarget( + name: "DeckOfPlayingCardsTests", + dependencies: ["DeckOfPlayingCards"]), + ] +) +``` + +When analyses is performed (e.g. `fossa analyze -o`), we will identify following as direct dependencies: +- https://github.com/apple/example-package-fisheryates.git +- https://github.com/apple/example-package-playingcard.git + +## F.A.Q + +### How do I *only perform analysis* for swift package dependencies? + +You can explicitly specify analyses target in `.fossa.yml` file. + +Example below, will exclude all analyses targets except swift. + +```yaml +# .fossa.yml + +version: 3 +targets: + only: + - type: swift +``` + +## References + +- [Swift Package Manager](https://github.com/apple/swift-package-manager) +- [Package.swift, must begin with version specifier](https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#about-the-swift-tools-version) +- [Package.swift, must be defined in single nested statement, and should not be modified after initialization](https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#package) \ No newline at end of file diff --git a/docs/userguide.md b/docs/userguide.md index 038eb2e4d..fc5f4ed3d 100644 --- a/docs/userguide.md +++ b/docs/userguide.md @@ -153,6 +153,7 @@ fossa analyze --help - [carthage](quickreference/carthage.md) - [cocoapods](quickreference/cocoapods.md) +- [swift package manager](quickreference/swiftpm.md) ## `fossa analyze` @@ -259,6 +260,7 @@ Supported dependency types: - `pub` - Dart dependencies found at [pub.dev](https://www.pub.dev/). - `pypi` - Python dependencies that are typically found at [Pypi.org](https://pypi.org/). - `cocoapods` - Swift and Objective-C dependencies found at [Cocoapods.org](https://cocoapods.org/). +- `swift` - Swift package manager dependencies. Specified as the full git repository `https://github.com/fossas/spectrometer`. - `url` - The URL type allows you to specify only the download location of an archive (e.g.: `.zip`, .`tar.gz`, etc.) in the `name` field and the FOSSA backend will attempt to download and scan it. Example for a github source dependency `https://github.com/fossas/spectrometer/archive/refs/tags/v2.7.2.tar.gz`. The `version` field will be silently ignored for `url` type dependencies. ### Custom dependencies diff --git a/spectrometer.cabal b/spectrometer.cabal index 73833373e..711c5044e 100644 --- a/spectrometer.cabal +++ b/spectrometer.cabal @@ -295,6 +295,8 @@ library Strategy.Ruby.BundleShow Strategy.Ruby.GemfileLock Strategy.Scala + Strategy.SwiftPM + Strategy.Swift.PackageSwift Strategy.Yarn Strategy.Yarn.V1.YarnLock Strategy.Yarn.V2.Lockfile @@ -391,6 +393,7 @@ test-suite unit-tests RPM.SpecFileSpec Ruby.BundleShowSpec Ruby.GemfileLockSpec + Swift.PackageSwiftSpec Yarn.V2.LockfileSpec Yarn.V2.ResolversSpec Yarn.YarnLockV1Spec diff --git a/src/App/Fossa/ManualDeps.hs b/src/App/Fossa/ManualDeps.hs index 4035ac30b..044ef15cc 100644 --- a/src/App/Fossa/ManualDeps.hs +++ b/src/App/Fossa/ManualDeps.hs @@ -253,6 +253,7 @@ depTypeFromText text = case text of "pypi" -> Just PipType "cocoapods" -> Just PodType "url" -> Just URLType + "swift" -> Just SwiftType _ -> Nothing -- unsupported dep, need to respond with an error and skip this dependency -- rpm is an unsupported type. This is because we currently have 2 RPM fetchers -- and we should wait for a need to determine which one to use for manually diff --git a/src/DepTypes.hs b/src/DepTypes.hs index 56b13f091..16cc7d6f1 100644 --- a/src/DepTypes.hs +++ b/src/DepTypes.hs @@ -95,6 +95,8 @@ data DepType HackageType | -- | A Carthage dependency -- effectively a "git" dependency. Name is repo path and version is tag/branch/hash CarthageType + | -- | A Swift Package Dependency -- effectively a "git" dependency. Name is repo path and version is tag/branch/hash + SwiftType deriving (Eq, Ord, Show, Generic) data VerConstraint diff --git a/src/Srclib/Converter.hs b/src/Srclib/Converter.hs index 0e17b6d23..6cbebc10e 100644 --- a/src/Srclib/Converter.hs +++ b/src/Srclib/Converter.hs @@ -121,6 +121,7 @@ depTypeToFetcher = \case URLType -> "url" UserType -> "user" PubType -> "pub" + SwiftType -> "swift" -- | GooglesourceType and SubprojectType are not supported with this function, since they're ambiguous. fetcherToDepType :: Text -> Maybe DepType diff --git a/src/Strategy/Swift/PackageSwift.hs b/src/Strategy/Swift/PackageSwift.hs new file mode 100644 index 000000000..83387a3f2 --- /dev/null +++ b/src/Strategy/Swift/PackageSwift.hs @@ -0,0 +1,248 @@ +module Strategy.Swift.PackageSwift ( + analyzePackageSwift, + + -- * for testing, + buildGraph, + parsePackageSwiftFile, + SwiftPackage (..), + SwiftPackageDep (..), + SwiftPackageGitDep (..), +) where + +import Control.Applicative (Alternative ((<|>)), optional) +import Control.Effect.Diagnostics (Diagnostics, context) +import Control.Monad (void) +import Control.Monad.Identity (Identity) +import Data.Foldable (asum) +import Data.Map.Strict qualified as Map +import Data.Maybe (isJust) +import Data.Text (Text) +import Data.Void (Void) +import DepTypes (DepType (GitType, SwiftType), Dependency (..), VerConstraint (CEq)) +import Effect.ReadFS (Has, ReadFS, readContentsParser) +import Graphing (Graphing, directs, induceJust) +import Path +import Text.Megaparsec ( + MonadParsec (takeWhile1P, try), + Parsec, + ParsecT, + anySingle, + between, + empty, + sepEndBy, + skipManyTill, + ) +import Text.Megaparsec.Char (space1) +import Text.Megaparsec.Char.Lexer qualified as Lexer +import Types (GraphBreadth (..)) + +-- | Parsing +-- * +type Parser = Parsec Void Text + +sc :: Parser () +sc = + Lexer.space + space1 + (Lexer.skipLineComment "//") + (Lexer.skipBlockComment "/*" "*/") + +lexeme :: Parser a -> Parser a +lexeme = Lexer.lexeme sc + +symbol :: Text -> Parser Text +symbol = Lexer.symbol sc + +scWOComment :: Parser () +scWOComment = Lexer.space space1 empty empty + +symbolWOComment :: Text -> Parser Text +symbolWOComment = Lexer.symbol scWOComment + +betweenDoubleQuotes :: Parser a -> Parser a +betweenDoubleQuotes = between (symbol "\"") (symbol "\"") + +betweenSquareBrackets :: Parser a -> Parser a +betweenSquareBrackets = between (symbol "[") (symbol "]") + +betweenBrackets :: Parser a -> Parser a +betweenBrackets = between (symbol "(") (symbol ")") + +maybeComma :: Parser () +maybeComma = void $ optional $ lexeme $ symbol "," + +parseQuotedText :: Parser Text +parseQuotedText = betweenDoubleQuotes (lexeme $ takeWhile1P (Just "quoted text") (/= '"')) + +parseKeyValue :: Text -> Parser a -> Parser a +parseKeyValue t parser = lexeme $ symbol (t <> ":") *> parser + +isEndLine :: Char -> Bool +isEndLine '\n' = True +isEndLine '\r' = True +isEndLine _ = False + +-- | Represents https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#methods. +data SwiftPackage = SwiftPackage + { swiftToolVersion :: Text + , packageDependencies :: [SwiftPackageDep] + } + deriving (Show, Eq, Ord) + +-- | Represents https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#package-dependency. +data SwiftPackageDep + = GitSource SwiftPackageGitDep + | PathSource Text + deriving (Show, Eq, Ord) + +data SwiftPackageGitDep = SwiftPackageGitDep + { srcOf :: Text + , branchOf :: Maybe Text + , revisionOf :: Maybe Text + , fromOf :: Maybe Text + , exactOf :: Maybe Text + , upToNextMajorOf :: Maybe Text + , upToNextMinorOf :: Maybe Text + , closedInterval :: Maybe (Text, Text) + , rhsHalfOpenInterval :: Maybe (Text, Text) + } + deriving (Show, Eq, Ord) + +parsePackageDep :: Parser SwiftPackageDep +parsePackageDep = try parsePathDep <|> parseGitDep + where + parsePathDep :: Parser SwiftPackageDep + parsePathDep = PathSource <$> (symbol ".package" *> betweenBrackets (parseKeyValue "path" parseQuotedText)) + + parseRequirement :: Text -> Parser Text + parseRequirement t = + try (symbol ("." <> t) *> betweenBrackets parseQuotedText) + <|> parseKeyValue t parseQuotedText + + parseUpToOperator :: Text -> Parser Text + parseUpToOperator t = symbol ("." <> t) *> betweenBrackets (parseRequirement "from") + + parseRange :: Text -> Parser (Text, Text) + parseRange rangeOperator = do + lhs <- parseQuotedText + _ <- symbol rangeOperator + rhs <- parseQuotedText + pure (lhs, rhs) + + optionallyTry :: ParsecT Void Text Identity a -> ParsecT Void Text Identity (Maybe a) + optionallyTry p = optional . try $ p <* maybeComma + + parseGitDep :: Parser SwiftPackageDep + parseGitDep = do + _ <- symbol ".package" <* symbol "(" + _ <- optionallyTry (parseKeyValue "name" parseQuotedText) + + -- Url (Required Field) + url <- parseKeyValue "url" $ parseQuotedText <* maybeComma + + -- Version Constraint (Optional Fields) + revision <- optionallyTry $ parseRequirement "revision" + branch <- optionallyTry $ parseRequirement "branch" + from <- optionallyTry $ parseRequirement "from" + exact <- optionallyTry $ parseRequirement "exact" + upToMajor <- optionallyTry $ parseUpToOperator "upToNextMajor" + upToMinor <- optionallyTry $ parseUpToOperator "upToNextMinor" + closedInterval <- optionallyTry $ parseRange "..." + rhsHalfOpenRange <- optionallyTry $ parseRange "..<" + + _ <- symbol ")" + pure $ + GitSource $ + SwiftPackageGitDep + url + branch + revision + from + exact + upToMajor + upToMinor + closedInterval + rhsHalfOpenRange + +parsePackageDependencies :: Parser [SwiftPackageDep] +parsePackageDependencies = do + _ <- lexeme $ skipManyTill anySingle $ symbol "let package = Package(" + skipManyTill anySingle (symbol "dependencies:") *> betweenSquareBrackets (sepEndBy (lexeme parsePackageDep) $ symbol ",") + +parseSwiftToolVersion :: Parser Text +parseSwiftToolVersion = + symbolWOComment "//" + *> parseKeyValue + "swift-tools-version" + (takeWhile1P (Just "swift-tools-version") $ not . isEndLine) + +parsePackageSwiftFile :: Parser SwiftPackage +parsePackageSwiftFile = do + -- Package.swift must specify version for swift tools + -- https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#about-the-swift-tools-version + swiftToolVersion <- parseSwiftToolVersion + SwiftPackage swiftToolVersion <$> parsePackageDependencies + +-- | Analysis +-- * +analyzePackageSwift :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> m (Graphing.Graphing Dependency, GraphBreadth) +analyzePackageSwift manifestFile = do + manifestContent <- context "Identifying dependencies in Package.swift" $ readContentsParser parsePackageSwiftFile manifestFile + graph <- context "Building dependency graph" $ pure $ buildGraph manifestContent + pure (graph, Partial) + +-- | Graph Building +-- * +buildGraph :: SwiftPackage -> Graphing.Graphing Dependency +buildGraph pkg = induceJust $ directs (map toDependency $ packageDependencies pkg) + +toDependency :: SwiftPackageDep -> Maybe Dependency +toDependency (GitSource pkgDep) = + Just $ + Dependency + { dependencyType = depType + , dependencyName = srcOf pkgDep + , dependencyVersion = + CEq + <$> asum + [ branchOf pkgDep + , revisionOf pkgDep + , exactOf pkgDep + , toFromExpression <$> fromOf pkgDep + , toUpToNextMajorExpression <$> upToNextMajorOf pkgDep + , toUpToNextMinorExpression <$> upToNextMinorOf pkgDep + , toClosedIntervalExpression <$> closedInterval pkgDep + , toRhsHalfOpenIntervalExpression <$> rhsHalfOpenInterval pkgDep + ] + , dependencyLocations = [] + , dependencyEnvironments = [] + , dependencyTags = Map.empty + } + where + -- from constraint is equivalent to upToNextMajor + -- Reference: https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#methods-3 + toFromExpression :: Text -> Text + toFromExpression = toUpToNextMajorExpression + + -- Fetcher accepts ~ operator, to perform + -- upToNext minor constraint validation. + toUpToNextMajorExpression :: Text -> Text + toUpToNextMajorExpression v = "^" <> v + + -- Fetcher accepts ~ operator, to perform + -- upToNext minor constraint validation. + toUpToNextMinorExpression :: Text -> Text + toUpToNextMinorExpression v = "~" <> v + + toClosedIntervalExpression :: (Text, Text) -> Text + toClosedIntervalExpression (lhs, rhs) = ">=" <> lhs <> " " <> "<=" <> rhs + + toRhsHalfOpenIntervalExpression :: (Text, Text) -> Text + toRhsHalfOpenIntervalExpression (lhs, rhs) = ">=" <> lhs <> " " <> "<" <> rhs + + depType :: DepType + depType = + if isJust $ asum [branchOf pkgDep, revisionOf pkgDep, exactOf pkgDep] + then GitType + else SwiftType +toDependency (PathSource _) = Nothing diff --git a/src/Strategy/SwiftPM.hs b/src/Strategy/SwiftPM.hs new file mode 100644 index 000000000..02bd09c34 --- /dev/null +++ b/src/Strategy/SwiftPM.hs @@ -0,0 +1,55 @@ +module Strategy.SwiftPM ( + discover, + findProjects, + mkProject, +) where + +import Control.Carrier.Simple (Has) +import Control.Effect.Diagnostics (Diagnostics, context) +import Discovery.Walk ( + WalkStep (WalkContinue), + findFileNamed, + walk', + ) +import Effect.ReadFS (ReadFS) +import Path +import Strategy.Swift.PackageSwift (analyzePackageSwift) +import Types (DependencyResults (..), DiscoveredProject (..)) + +data SwiftPackageProject = SwiftPackageProject + { manifest :: Path Abs File + , projectDir :: Path Abs Dir + , resolved :: Maybe (Path Abs File) + } + deriving (Show, Eq, Ord) + +discover :: (Has ReadFS sig m, Has Diagnostics sig m, Has ReadFS rsig run, Has Diagnostics rsig run) => Path Abs Dir -> m [DiscoveredProject run] +discover dir = context "Swift" $ do + projects <- context "Finding projects" $ findProjects dir + pure (map mkProject projects) + +findProjects :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m [SwiftPackageProject] +findProjects = walk' $ \dir _ files -> do + case findFileNamed "Package.swift" files of + Nothing -> pure ([], WalkContinue) + Just file -> pure ([SwiftPackageProject file dir Nothing], WalkContinue) + +mkProject :: (Has ReadFS sig n, Has Diagnostics sig n) => SwiftPackageProject -> DiscoveredProject n +mkProject project = + DiscoveredProject + { projectType = "swift" + , projectBuildTargets = mempty + , projectDependencyResults = const $ getDeps project + , projectPath = projectDir project + , projectLicenses = pure [] + } + +getDeps :: (Has ReadFS sig m, Has Diagnostics sig m) => SwiftPackageProject -> m DependencyResults +getDeps project = do + (graph, graphBreadth) <- analyzePackageSwift $ manifest project + pure $ + DependencyResults + { dependencyGraph = graph + , dependencyGraphBreadth = graphBreadth + , dependencyManifestFiles = [manifest project] + } diff --git a/test/Swift/PackageSwiftSpec.hs b/test/Swift/PackageSwiftSpec.hs new file mode 100644 index 000000000..4f8f04099 --- /dev/null +++ b/test/Swift/PackageSwiftSpec.hs @@ -0,0 +1,125 @@ +module Swift.PackageSwiftSpec ( + spec, +) where + +import Data.Map.Strict qualified as Map +import Data.Text (Text) +import Data.Text.IO qualified as TIO +import DepTypes (DepType (GitType, SwiftType), Dependency (..), VerConstraint (CEq)) +import GraphUtil (expectDeps, expectDirect, expectEdges) +import Strategy.Swift.PackageSwift ( + SwiftPackage (..), + SwiftPackageDep (..), + SwiftPackageGitDep (..), + buildGraph, + parsePackageSwiftFile, + ) +import Test.Hspec +import Text.Megaparsec (runParser) + +gitDepWithoutConstraint :: Text -> SwiftPackageGitDep +gitDepWithoutConstraint url = SwiftPackageGitDep url Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing + +gitDepWithBranch :: Text -> Text -> SwiftPackageGitDep +gitDepWithBranch url branch = (gitDepWithoutConstraint url){branchOf = Just branch} + +gitDepWithRevision :: Text -> Text -> SwiftPackageGitDep +gitDepWithRevision url revision = (gitDepWithoutConstraint url){revisionOf = Just revision} + +gitDepFrom :: Text -> Text -> SwiftPackageGitDep +gitDepFrom url from = (gitDepWithoutConstraint url){fromOf = Just from} + +gitDepExactly :: Text -> Text -> SwiftPackageGitDep +gitDepExactly url exact = (gitDepWithoutConstraint url){exactOf = Just exact} + +gitDepUpToNextMajor :: Text -> Text -> SwiftPackageGitDep +gitDepUpToNextMajor url constraint = (gitDepWithoutConstraint url){upToNextMajorOf = Just constraint} + +gitDepUpToNextMinor :: Text -> Text -> SwiftPackageGitDep +gitDepUpToNextMinor url constraint = (gitDepWithoutConstraint url){upToNextMinorOf = Just constraint} + +gitDepWithClosedRange :: Text -> Text -> Text -> SwiftPackageGitDep +gitDepWithClosedRange url lhs rhs = (gitDepWithoutConstraint url){closedInterval = Just (lhs, rhs)} + +gitDepWithRhsHalfOpenInterval :: Text -> Text -> Text -> SwiftPackageGitDep +gitDepWithRhsHalfOpenInterval url lhs rhs = (gitDepWithoutConstraint url){rhsHalfOpenInterval = Just (lhs, rhs)} + +expectedSwiftPackage :: SwiftPackage +expectedSwiftPackage = + SwiftPackage "5.3" $ + map + GitSource + [ -- without any constraint + gitDepWithoutConstraint "https://github.com/kirualex/SwiftyGif.git" + , -- from + gitDepFrom "https://github.com/apple/example-package-playingcard.git" "3.0.0" + , gitDepFrom "https://github.com/kaishin/Gifu.git" "3.2.2" + , -- exact + gitDepExactly "https://github.com/kelvin13/jpeg.git" "1.0.0" + , gitDepExactly "https://github.com/shogo4405/HaishinKit.swift" "1.1.6" + , -- upTo constraint + gitDepUpToNextMajor "https://github.com/dankogai/swift-sion" "0.0.1" + , gitDepUpToNextMinor "git@github.com:behrang/YamlSwift.git" "3.4.0" + , -- branch + gitDepWithBranch "https://github.com/vapor/vapor" "main" + , gitDepWithBranch "git@github.com:vapor-community/HTMLKit.git" "function-builder" + , -- revision + gitDepWithRevision "https://github.com/SwiftyBeaver/SwiftyBeaver.git" "607fc8d64388652135f4dcf6a1a340e3a0641088" + , gitDepWithRevision "https://github.com/roberthein/TinyConstraints.git" "3262e5c591d4ab6272255df2087a01bbebd138dc" + , -- range + gitDepWithRhsHalfOpenInterval "https://github.com/LeoNatan/LNPopupController.git" "2.5.0" "2.5.6" + , gitDepWithClosedRange "https://github.com/Polidea/RxBluetoothKit.git" "3.0.5" "3.0.7" + ] + +spec :: Spec +spec = do + packageDotSwiftFile <- runIO (TIO.readFile "test/Swift/testdata/Package.Swift") + + describe "Parses Package.swift file" $ do + it "should parse swift-tools-version" $ do + case runParser parsePackageSwiftFile "" packageDotSwiftFile of + Left failCode -> expectationFailure $ show failCode + Right result -> result `shouldBe` expectedSwiftPackage + + describe "buildGraph" $ do + it "should use git dependency type, when constraint is of branch, revision, or exact type" $ do + let expectedDeps = + [ Dependency GitType "some-url" (CEq <$> Just "some-ref") [] [] Map.empty + , Dependency GitType "some-url" (CEq <$> Just "some-branch") [] [] Map.empty + , Dependency GitType "some-url" (CEq <$> Just "1.0.0") [] [] Map.empty + ] + let graph = + buildGraph $ + SwiftPackage + "5.3" + [ GitSource $ gitDepWithRevision "some-url" "some-ref" + , GitSource $ gitDepWithBranch "some-url" "some-branch" + , GitSource $ gitDepExactly "some-url" "1.0.0" + ] + expectDirect expectedDeps graph + expectDeps expectedDeps graph + expectEdges [] graph + + it "should use swift dependency type, when constraint uses follows, range, upToNextMajor, or upToNextMinor" $ do + let graph = + buildGraph $ + SwiftPackage + "5.3" + [ GitSource $ gitDepWithoutConstraint "some-url-dep" + , GitSource $ gitDepFrom "some-url-dep" "3.0.0" + , GitSource $ gitDepUpToNextMajor "some-url-dep" "2.0.0" + , GitSource $ gitDepUpToNextMinor "some-url-dep" "1.0.0" + , GitSource $ gitDepWithRhsHalfOpenInterval "some-url-dep" "2.5.0" "2.5.6" + , GitSource $ gitDepWithClosedRange "some-url-dep" "3.0.5" "3.0.7" + ] + let expectedDeps = + [ Dependency SwiftType "some-url-dep" Nothing [] [] Map.empty + , Dependency SwiftType "some-url-dep" (CEq <$> Just "^3.0.0") [] [] Map.empty + , Dependency SwiftType "some-url-dep" (CEq <$> Just "^2.0.0") [] [] Map.empty + , Dependency SwiftType "some-url-dep" (CEq <$> Just "~1.0.0") [] [] Map.empty + , Dependency SwiftType "some-url-dep" (CEq <$> Just ">=2.5.0 <2.5.6") [] [] Map.empty + , Dependency SwiftType "some-url-dep" (CEq <$> Just ">=3.0.5 <=3.0.7") [] [] Map.empty + ] + expectDirect expectedDeps graph + expectDeps expectedDeps graph + expectEdges [] graph diff --git a/test/Swift/testdata/Package.swift b/test/Swift/testdata/Package.swift new file mode 100644 index 000000000..3ab9f51ac --- /dev/null +++ b/test/Swift/testdata/Package.swift @@ -0,0 +1,64 @@ +// swift-tools-version:5.3 + +import PackageDescription + +let package = Package( + name: "DeckOfPlayingCardsss", + defaultLocalization: "en", + platforms: [ + .macOS(.v10_15), + ], + products: [ + .executable(name: "tool", targets: ["tool"]), + .library(name: "Paper", targets: ["Paper"]), + .library(name: "PaperStatic", type: .static, targets: ["Paper"]), + .library(name: "PaperDynamic", type: .dynamic, targets: ["Paper"]), + ], + dependencies: [ + + // without any contsraint + .package(url: "https://github.com/kirualex/SwiftyGif.git"), + + // pkg version + .package( + name: "PlayingCard", + url: "https://github.com/apple/example-package-playingcard.git", + from: "3.0.0" + ), + .package(url: "https://github.com/kaishin/Gifu.git", .from("3.2.2")), + + // exact + .package(url: "https://github.com/kelvin13/jpeg.git", .exact("1.0.0")), + .package(url: "https://github.com/shogo4405/HaishinKit.swift", exact: "1.1.6"), + + // upToNextMajor + .package(url: "https://github.com/dankogai/swift-sion", .upToNextMajor(from: "0.0.1")), + + // upToNextMinor + .package(url: "git@github.com:behrang/YamlSwift.git", .upToNextMinor(from: "3.4.0")), + + // branch + .package(url: "https://github.com/vapor/vapor", .branch("main")), + .package(url: "git@github.com:vapor-community/HTMLKit.git", branch: "function-builder"), + + // revision + .package(url: "https://github.com/SwiftyBeaver/SwiftyBeaver.git", revision: "607fc8d64388652135f4dcf6a1a340e3a0641088"), + .package(url: "https://github.com/roberthein/TinyConstraints.git", .revision("3262e5c591d4ab6272255df2087a01bbebd138dc")), + + // range + .package(url: "https://github.com/LeoNatan/LNPopupController.git", "2.5.0"..<"2.5.6"), + .package(url: "https://github.com/Polidea/RxBluetoothKit.git", "3.0.5"..."3.0.7"), + ], + targets: [ + .target( + name: "DeckOfPlayingCards", + dependencies: [ + .byName(name: "PlayingCard") + ]), + .testTarget( + name: "DeckOfPlayingCardsTests", + dependencies: [ + .target(name: "DeckOfPlayingCards") + ]), + ] +) \ No newline at end of file From 3da750cad8c5e78e08439aebf816b65386f4fc4e Mon Sep 17 00:00:00 2001 From: Megh Date: Tue, 31 Aug 2021 19:28:04 -0600 Subject: [PATCH 2/9] Includes swift pm in discovery --- src/App/Fossa/Analyze.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index f70f241e6..4cc5801ed 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -98,6 +98,7 @@ import Strategy.Python.Setuptools qualified as Setuptools import Strategy.RPM qualified as RPM import Strategy.Rebar3 qualified as Rebar3 import Strategy.Scala qualified as Scala +import Strategy.SwiftPM qualified as SwiftPM import Strategy.Yarn qualified as Yarn import System.Exit (die) import Types (DiscoveredProject (..), FoundTargets) @@ -222,6 +223,7 @@ discoverFuncs = , Scala.discover , Setuptools.discover , Stack.discover + , SwiftPM.discover , Yarn.discover ] From 18b01a8d0b1a2d87225e4e74d59d6fb29d8b6009 Mon Sep 17 00:00:00 2001 From: Megh Date: Tue, 31 Aug 2021 19:36:06 -0600 Subject: [PATCH 3/9] fixes failing test --- test/Swift/PackageSwiftSpec.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Swift/PackageSwiftSpec.hs b/test/Swift/PackageSwiftSpec.hs index 4f8f04099..fe281397b 100644 --- a/test/Swift/PackageSwiftSpec.hs +++ b/test/Swift/PackageSwiftSpec.hs @@ -73,7 +73,7 @@ expectedSwiftPackage = spec :: Spec spec = do - packageDotSwiftFile <- runIO (TIO.readFile "test/Swift/testdata/Package.Swift") + packageDotSwiftFile <- runIO (TIO.readFile "test/Swift/testdata/Package.swift") describe "Parses Package.swift file" $ do it "should parse swift-tools-version" $ do From 1fde83be078f03aeec4cc97ee53f74df2befb28f Mon Sep 17 00:00:00 2001 From: Megh Date: Tue, 31 Aug 2021 19:58:06 -0600 Subject: [PATCH 4/9] minor docs fix --- src/Strategy/Swift/PackageSwift.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Strategy/Swift/PackageSwift.hs b/src/Strategy/Swift/PackageSwift.hs index 83387a3f2..b86a9b7c7 100644 --- a/src/Strategy/Swift/PackageSwift.hs +++ b/src/Strategy/Swift/PackageSwift.hs @@ -224,8 +224,8 @@ toDependency (GitSource pkgDep) = toFromExpression :: Text -> Text toFromExpression = toUpToNextMajorExpression - -- Fetcher accepts ~ operator, to perform - -- upToNext minor constraint validation. + -- Fetcher accepts ^ operator, to perform + -- upToNext major constraint validation. toUpToNextMajorExpression :: Text -> Text toUpToNextMajorExpression v = "^" <> v From e0e782af4fa2e5af5d7bff09732fb87cdb1e9602 Mon Sep 17 00:00:00 2001 From: Megh Date: Thu, 2 Sep 2021 21:21:38 -0600 Subject: [PATCH 5/9] applies pr feedback --- src/Strategy/Swift/PackageSwift.hs | 145 ++++++++++++++--------------- src/Strategy/SwiftPM.hs | 6 +- test/Swift/PackageSwiftSpec.hs | 19 ++-- 3 files changed, 81 insertions(+), 89 deletions(-) diff --git a/src/Strategy/Swift/PackageSwift.hs b/src/Strategy/Swift/PackageSwift.hs index b86a9b7c7..fe9bb3717 100644 --- a/src/Strategy/Swift/PackageSwift.hs +++ b/src/Strategy/Swift/PackageSwift.hs @@ -7,15 +7,14 @@ module Strategy.Swift.PackageSwift ( SwiftPackage (..), SwiftPackageDep (..), SwiftPackageGitDep (..), + SwiftPackageGitDepRequirement (..), ) where import Control.Applicative (Alternative ((<|>)), optional) import Control.Effect.Diagnostics (Diagnostics, context) import Control.Monad (void) -import Control.Monad.Identity (Identity) import Data.Foldable (asum) import Data.Map.Strict qualified as Map -import Data.Maybe (isJust) import Data.Text (Text) import Data.Void (Void) import DepTypes (DepType (GitType, SwiftType), Dependency (..), VerConstraint (CEq)) @@ -25,7 +24,6 @@ import Path import Text.Megaparsec ( MonadParsec (takeWhile1P, try), Parsec, - ParsecT, anySingle, between, empty, @@ -34,7 +32,6 @@ import Text.Megaparsec ( ) import Text.Megaparsec.Char (space1) import Text.Megaparsec.Char.Lexer qualified as Lexer -import Types (GraphBreadth (..)) -- | Parsing -- * @@ -97,17 +94,39 @@ data SwiftPackageDep data SwiftPackageGitDep = SwiftPackageGitDep { srcOf :: Text - , branchOf :: Maybe Text - , revisionOf :: Maybe Text - , fromOf :: Maybe Text - , exactOf :: Maybe Text - , upToNextMajorOf :: Maybe Text - , upToNextMinorOf :: Maybe Text - , closedInterval :: Maybe (Text, Text) - , rhsHalfOpenInterval :: Maybe (Text, Text) + , versionRequirement :: Maybe SwiftPackageGitDepRequirement } deriving (Show, Eq, Ord) +data SwiftPackageGitDepRequirement + = Branch Text + | Revision Text + | Exact Text + | From Text + | UpToNextMajor Text + | UpToNextMinor Text + | ClosedInterval (Text, Text) + | RhsHalfOpenInterval (Text, Text) + deriving (Show, Eq, Ord) + +toConstraint :: SwiftPackageGitDepRequirement -> VerConstraint +toConstraint (Branch b) = CEq b +toConstraint (Revision r) = CEq r +toConstraint (Exact e) = CEq e +-- from constraint is equivalent to upToNextMajor +-- Reference: https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#methods-3 +toConstraint (From f) = CEq $ "^" <> f +toConstraint (UpToNextMajor c) = CEq $ "^" <> c +toConstraint (UpToNextMinor c) = CEq $ "~" <> c +toConstraint (ClosedInterval (lhs, rhs)) = CEq $ ">=" <> lhs <> " " <> "<=" <> rhs +toConstraint (RhsHalfOpenInterval (lhs, rhs)) = CEq $ ">=" <> lhs <> " " <> "<" <> rhs + +isGitRefConstraint :: SwiftPackageGitDepRequirement -> Bool +isGitRefConstraint (Branch _) = True +isGitRefConstraint (Revision _) = True +isGitRefConstraint (Exact _) = True +isGitRefConstraint _ = False + parsePackageDep :: Parser SwiftPackageDep parsePackageDep = try parsePathDep <|> parseGitDep where @@ -129,7 +148,7 @@ parsePackageDep = try parsePathDep <|> parseGitDep rhs <- parseQuotedText pure (lhs, rhs) - optionallyTry :: ParsecT Void Text Identity a -> ParsecT Void Text Identity (Maybe a) + optionallyTry :: Parser a -> Parser (Maybe a) optionallyTry p = optional . try $ p <* maybeComma parseGitDep :: Parser SwiftPackageDep @@ -140,29 +159,22 @@ parsePackageDep = try parsePathDep <|> parseGitDep -- Url (Required Field) url <- parseKeyValue "url" $ parseQuotedText <* maybeComma - -- Version Constraint (Optional Fields) - revision <- optionallyTry $ parseRequirement "revision" - branch <- optionallyTry $ parseRequirement "branch" - from <- optionallyTry $ parseRequirement "from" - exact <- optionallyTry $ parseRequirement "exact" - upToMajor <- optionallyTry $ parseUpToOperator "upToNextMajor" - upToMinor <- optionallyTry $ parseUpToOperator "upToNextMinor" - closedInterval <- optionallyTry $ parseRange "..." - rhsHalfOpenRange <- optionallyTry $ parseRange "..<" - + versionRequirement <- + optional $ + asum $ + map + try + [ Revision <$> parseRequirement "revision" + , Branch <$> parseRequirement "branch" + , From <$> parseRequirement "from" + , Exact <$> parseRequirement "exact" + , UpToNextMajor <$> parseUpToOperator "upToNextMajor" + , UpToNextMinor <$> parseUpToOperator "upToNextMinor" + , ClosedInterval <$> parseRange "..." + , RhsHalfOpenInterval <$> parseRange "..<" + ] _ <- symbol ")" - pure $ - GitSource $ - SwiftPackageGitDep - url - branch - revision - from - exact - upToMajor - upToMinor - closedInterval - rhsHalfOpenRange + pure $ GitSource $ SwiftPackageGitDep url (versionRequirement) parsePackageDependencies :: Parser [SwiftPackageDep] parsePackageDependencies = do @@ -185,11 +197,19 @@ parsePackageSwiftFile = do -- | Analysis -- * -analyzePackageSwift :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> m (Graphing.Graphing Dependency, GraphBreadth) -analyzePackageSwift manifestFile = do - manifestContent <- context "Identifying dependencies in Package.swift" $ readContentsParser parsePackageSwiftFile manifestFile - graph <- context "Building dependency graph" $ pure $ buildGraph manifestContent - pure (graph, Partial) +analyzePackageSwift :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> m (Graphing.Graphing Dependency) +analyzePackageSwift manifestFile = + do + manifestContent <- + context + "Identifying dependencies in Package.swift" + $ readContentsParser parsePackageSwiftFile manifestFile + context "Building dependency graph" $ + pure $ buildGraph manifestContent + +-- manifestContent <- context "Identifying dependencies in Package.swift" $ readContentsParser parsePackageSwiftFile manifestFile +-- graph <- context "Building dependency graph" $ pure $ buildGraph manifestContent +-- pure graph -- | Graph Building -- * @@ -197,52 +217,23 @@ buildGraph :: SwiftPackage -> Graphing.Graphing Dependency buildGraph pkg = induceJust $ directs (map toDependency $ packageDependencies pkg) toDependency :: SwiftPackageDep -> Maybe Dependency +toDependency (PathSource _) = Nothing toDependency (GitSource pkgDep) = Just $ Dependency { dependencyType = depType , dependencyName = srcOf pkgDep - , dependencyVersion = - CEq - <$> asum - [ branchOf pkgDep - , revisionOf pkgDep - , exactOf pkgDep - , toFromExpression <$> fromOf pkgDep - , toUpToNextMajorExpression <$> upToNextMajorOf pkgDep - , toUpToNextMinorExpression <$> upToNextMinorOf pkgDep - , toClosedIntervalExpression <$> closedInterval pkgDep - , toRhsHalfOpenIntervalExpression <$> rhsHalfOpenInterval pkgDep - ] + , dependencyVersion = toConstraint <$> versionRequirement pkgDep , dependencyLocations = [] , dependencyEnvironments = [] , dependencyTags = Map.empty } where - -- from constraint is equivalent to upToNextMajor - -- Reference: https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#methods-3 - toFromExpression :: Text -> Text - toFromExpression = toUpToNextMajorExpression - - -- Fetcher accepts ^ operator, to perform - -- upToNext major constraint validation. - toUpToNextMajorExpression :: Text -> Text - toUpToNextMajorExpression v = "^" <> v - - -- Fetcher accepts ~ operator, to perform - -- upToNext minor constraint validation. - toUpToNextMinorExpression :: Text -> Text - toUpToNextMinorExpression v = "~" <> v - - toClosedIntervalExpression :: (Text, Text) -> Text - toClosedIntervalExpression (lhs, rhs) = ">=" <> lhs <> " " <> "<=" <> rhs - - toRhsHalfOpenIntervalExpression :: (Text, Text) -> Text - toRhsHalfOpenIntervalExpression (lhs, rhs) = ">=" <> lhs <> " " <> "<" <> rhs - depType :: DepType depType = - if isJust $ asum [branchOf pkgDep, revisionOf pkgDep, exactOf pkgDep] - then GitType - else SwiftType -toDependency (PathSource _) = Nothing + case isGitRefConstraint <$> versionRequirement pkgDep of + Just True -> GitType + Just False -> SwiftType + -- We want to select highest priority tag (descending with semver versioning) + -- instead of HEAD of the repository + Nothing -> SwiftType diff --git a/src/Strategy/SwiftPM.hs b/src/Strategy/SwiftPM.hs index 02bd09c34..ab4bf64e1 100644 --- a/src/Strategy/SwiftPM.hs +++ b/src/Strategy/SwiftPM.hs @@ -14,7 +14,7 @@ import Discovery.Walk ( import Effect.ReadFS (ReadFS) import Path import Strategy.Swift.PackageSwift (analyzePackageSwift) -import Types (DependencyResults (..), DiscoveredProject (..)) +import Types (DependencyResults (..), DiscoveredProject (..), GraphBreadth (..)) data SwiftPackageProject = SwiftPackageProject { manifest :: Path Abs File @@ -46,10 +46,10 @@ mkProject project = getDeps :: (Has ReadFS sig m, Has Diagnostics sig m) => SwiftPackageProject -> m DependencyResults getDeps project = do - (graph, graphBreadth) <- analyzePackageSwift $ manifest project + graph <- analyzePackageSwift $ manifest project pure $ DependencyResults { dependencyGraph = graph - , dependencyGraphBreadth = graphBreadth + , dependencyGraphBreadth = Partial , dependencyManifestFiles = [manifest project] } diff --git a/test/Swift/PackageSwiftSpec.hs b/test/Swift/PackageSwiftSpec.hs index fe281397b..144442b72 100644 --- a/test/Swift/PackageSwiftSpec.hs +++ b/test/Swift/PackageSwiftSpec.hs @@ -11,6 +11,7 @@ import Strategy.Swift.PackageSwift ( SwiftPackage (..), SwiftPackageDep (..), SwiftPackageGitDep (..), + SwiftPackageGitDepRequirement (..), buildGraph, parsePackageSwiftFile, ) @@ -18,31 +19,31 @@ import Test.Hspec import Text.Megaparsec (runParser) gitDepWithoutConstraint :: Text -> SwiftPackageGitDep -gitDepWithoutConstraint url = SwiftPackageGitDep url Nothing Nothing Nothing Nothing Nothing Nothing Nothing Nothing +gitDepWithoutConstraint url = SwiftPackageGitDep url Nothing gitDepWithBranch :: Text -> Text -> SwiftPackageGitDep -gitDepWithBranch url branch = (gitDepWithoutConstraint url){branchOf = Just branch} +gitDepWithBranch url branch = (gitDepWithoutConstraint url){versionRequirement = Just $ Branch branch} gitDepWithRevision :: Text -> Text -> SwiftPackageGitDep -gitDepWithRevision url revision = (gitDepWithoutConstraint url){revisionOf = Just revision} +gitDepWithRevision url revision = (gitDepWithoutConstraint url){versionRequirement = Just $ Revision revision} gitDepFrom :: Text -> Text -> SwiftPackageGitDep -gitDepFrom url from = (gitDepWithoutConstraint url){fromOf = Just from} +gitDepFrom url from = (gitDepWithoutConstraint url){versionRequirement = Just $ From from} gitDepExactly :: Text -> Text -> SwiftPackageGitDep -gitDepExactly url exact = (gitDepWithoutConstraint url){exactOf = Just exact} +gitDepExactly url exact = (gitDepWithoutConstraint url){versionRequirement = Just $ Exact exact} gitDepUpToNextMajor :: Text -> Text -> SwiftPackageGitDep -gitDepUpToNextMajor url constraint = (gitDepWithoutConstraint url){upToNextMajorOf = Just constraint} +gitDepUpToNextMajor url constraint = (gitDepWithoutConstraint url){versionRequirement = Just $ UpToNextMajor constraint} gitDepUpToNextMinor :: Text -> Text -> SwiftPackageGitDep -gitDepUpToNextMinor url constraint = (gitDepWithoutConstraint url){upToNextMinorOf = Just constraint} +gitDepUpToNextMinor url constraint = (gitDepWithoutConstraint url){versionRequirement = Just $ UpToNextMinor constraint} gitDepWithClosedRange :: Text -> Text -> Text -> SwiftPackageGitDep -gitDepWithClosedRange url lhs rhs = (gitDepWithoutConstraint url){closedInterval = Just (lhs, rhs)} +gitDepWithClosedRange url lhs rhs = (gitDepWithoutConstraint url){versionRequirement = Just $ ClosedInterval (lhs, rhs)} gitDepWithRhsHalfOpenInterval :: Text -> Text -> Text -> SwiftPackageGitDep -gitDepWithRhsHalfOpenInterval url lhs rhs = (gitDepWithoutConstraint url){rhsHalfOpenInterval = Just (lhs, rhs)} +gitDepWithRhsHalfOpenInterval url lhs rhs = (gitDepWithoutConstraint url){versionRequirement = Just $ RhsHalfOpenInterval (lhs, rhs)} expectedSwiftPackage :: SwiftPackage expectedSwiftPackage = From c082ef53319d25b548af6dd9bb08d2177052e7f6 Mon Sep 17 00:00:00 2001 From: Megh Date: Tue, 7 Sep 2021 11:28:35 -0600 Subject: [PATCH 6/9] removes dead comment --- src/Strategy/Swift/PackageSwift.hs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Strategy/Swift/PackageSwift.hs b/src/Strategy/Swift/PackageSwift.hs index fe9bb3717..205b9be0d 100644 --- a/src/Strategy/Swift/PackageSwift.hs +++ b/src/Strategy/Swift/PackageSwift.hs @@ -207,10 +207,6 @@ analyzePackageSwift manifestFile = context "Building dependency graph" $ pure $ buildGraph manifestContent --- manifestContent <- context "Identifying dependencies in Package.swift" $ readContentsParser parsePackageSwiftFile manifestFile --- graph <- context "Building dependency graph" $ pure $ buildGraph manifestContent --- pure graph - -- | Graph Building -- * buildGraph :: SwiftPackage -> Graphing.Graphing Dependency From b72ae5664c12cf723a6179d92b491d9ae6183499 Mon Sep 17 00:00:00 2001 From: Megh Date: Wed, 8 Sep 2021 20:37:04 -0600 Subject: [PATCH 7/9] applies pr feedback --- docs/strategies/ios/swift.md | 8 ++++---- src/Strategy/Swift/PackageSwift.hs | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/strategies/ios/swift.md b/docs/strategies/ios/swift.md index 762774620..6c02ab2e8 100644 --- a/docs/strategies/ios/swift.md +++ b/docs/strategies/ios/swift.md @@ -15,7 +15,7 @@ Find all swift manifest files, named: `Package.swift` ## Limitation -- Path dependencies are ignored in the analyses (e.g. `package(path: "./../local-pkg")`) +- Path dependencies are ignored in the analysis (e.g. `package(path: "./../local-pkg")`) ## Example @@ -45,7 +45,7 @@ let package = Package( ) ``` -When analyses is performed (e.g. `fossa analyze -o`), we will identify following as direct dependencies: +When analysis is performed (e.g. `fossa analyze -o`), we will identify following as direct dependencies: - https://github.com/apple/example-package-fisheryates.git - https://github.com/apple/example-package-playingcard.git @@ -53,9 +53,9 @@ When analyses is performed (e.g. `fossa analyze -o`), we will identify following ### How do I *only perform analysis* for swift package dependencies? -You can explicitly specify analyses target in `.fossa.yml` file. +You can explicitly specify analysis an target in `.fossa.yml` file. -Example below, will exclude all analyses targets except swift. +Example below, will exclude all analysis targets except swift. ```yaml # .fossa.yml diff --git a/src/Strategy/Swift/PackageSwift.hs b/src/Strategy/Swift/PackageSwift.hs index 205b9be0d..e1a9d8351 100644 --- a/src/Strategy/Swift/PackageSwift.hs +++ b/src/Strategy/Swift/PackageSwift.hs @@ -98,6 +98,7 @@ data SwiftPackageGitDep = SwiftPackageGitDep } deriving (Show, Eq, Ord) +-- | Represents https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#methods-3. data SwiftPackageGitDepRequirement = Branch Text | Revision Text @@ -109,6 +110,9 @@ data SwiftPackageGitDepRequirement | RhsHalfOpenInterval (Text, Text) deriving (Show, Eq, Ord) +-- Note: Swift fetcher is able to resolve, >=, <, <=, ^, ~ operators. +-- TODO: Leverage `VerConstraint` (CAnd, etc.) +-- TODO: Modify Srclib.Converter.verConstraintToRevision to transform constraint for fetcher toConstraint :: SwiftPackageGitDepRequirement -> VerConstraint toConstraint (Branch b) = CEq b toConstraint (Revision r) = CEq r From 0d7c0d47590c69213f6e0d2a11685769737b18c2 Mon Sep 17 00:00:00 2001 From: meghfossa <86321858+meghfossa@users.noreply.github.com> Date: Fri, 24 Sep 2021 16:34:52 -0600 Subject: [PATCH 8/9] Swift PM: Analyzes `Package.resolved` file (second of many) (#356) --- docs/strategies/ios/swift.md | 155 +++++++++++++---- spectrometer.cabal | 7 + src/App/Fossa/Analyze/Project.hs | 6 +- src/Strategy/Swift/PackageResolved.hs | 78 +++++++++ src/Strategy/Swift/PackageSwift.hs | 52 ++++-- src/Strategy/Swift/Xcode/Pbxproj.hs | 115 +++++++++++++ src/Strategy/Swift/Xcode/PbxprojParser.hs | 187 +++++++++++++++++++++ src/Strategy/SwiftPM.hs | 108 ++++++++++-- test/Swift/PackageResolvedSpec.hs | 41 +++++ test/Swift/PackageSwiftSpec.hs | 87 ++++++++-- test/Swift/Xcode/PbxprojParserSpec.hs | 193 ++++++++++++++++++++++ test/Swift/Xcode/PbxprojSpec.hs | 172 +++++++++++++++++++ test/Swift/Xcode/testdata/project.pbxproj | 125 ++++++++++++++ test/Swift/testdata/Package.resolved | 34 ++++ 14 files changed, 1279 insertions(+), 81 deletions(-) create mode 100644 src/Strategy/Swift/PackageResolved.hs create mode 100644 src/Strategy/Swift/Xcode/Pbxproj.hs create mode 100644 src/Strategy/Swift/Xcode/PbxprojParser.hs create mode 100644 test/Swift/PackageResolvedSpec.hs create mode 100644 test/Swift/Xcode/PbxprojParserSpec.hs create mode 100644 test/Swift/Xcode/PbxprojSpec.hs create mode 100644 test/Swift/Xcode/testdata/project.pbxproj create mode 100644 test/Swift/testdata/Package.resolved diff --git a/docs/strategies/ios/swift.md b/docs/strategies/ios/swift.md index 6c02ab2e8..49703481d 100644 --- a/docs/strategies/ios/swift.md +++ b/docs/strategies/ios/swift.md @@ -2,60 +2,131 @@ ## Project Discovery -Find all swift manifest files, named: `Package.swift` +Find all files named: `Package.swift` or find Xcode's project file named: `project.pbxproj`. +We will not scan `.build` directory if the `Package.swift` or Xcode project file is discovered. -# Swift Package Analysis +# Swift Analysis -| Strategy | Direct Deps | Deep Deps | Edges | Classifies Test Dependencies | -| --------------------------------------------- | ------------------ | --------- | ----- | ---------------------------- | -| parse package dependencies in `Package.swift` | :white_check_mark: | :x: | :x: | :x: | +| Strategy | Direct Deps | Deep Deps | Edges | Classifies Test Dependencies | +| ------------------------------------------------------------------------ | ------------------ | ------------------ | ----- | ---------------------------- | +| Parse dependencies from `Package.swift` | :white_check_mark: | :x: | :x: | :x: | +| Parse dependencies from `Package.swift` and `Package.resolved` | :white_check_mark: | :white_check_mark: | :x: | :x: | +| Parse dependencies from Xcode's `project.pbxproj` | :white_check_mark: | :x: | :x: | :x: | +| Parse dependencies from Xcode's `project.pbxproj` and `Package.resolved` | :white_check_mark: | :white_check_mark: | :x: | :x: | -- Manifest file - `Package.swift`, must begin with `// swift-tools-version:` string, followed by version number specifier. -- We follow swift package manager's convention, and presume properties of package are defined in a single nested initializer statement, and are not modified after initialization. +- Manifest file: `Package.swift`, must begin with `// swift-tools-version:` string, followed by version number specifier. +- We follow swift package manager's convention and presume properties of the package are defined in a single nested initializer statement and are not modified after initialization. +- Valid Xcode project for swift, is defined by the discovery of `project.pbxproj` file in ASCII plist format with at least one `XCRemoteSwiftPackageReference` object in its content. -## Limitation +## Limitations - Path dependencies are ignored in the analysis (e.g. `package(path: "./../local-pkg")`) +- If the Xcode project dependencies are sourced via a local path, they will be ignored in the analysis. +- Only Xcode project files in ASCII plist format with UTF-8 encoding are supported. -## Example +## Example Create Package.swift file in the directory. Add dependencies, targets, products, and source code. Example Package.swift file is shown below. By convention, the properties of a Package are defined in a single nested initializer statement, and not modified after initialization. ```swift -// swift-tools-version:4.0 +// swift-tools-version:5.4.0 import PackageDescription let package = Package( - name: "DeckOfPlayingCards", - products: [ - .library(name: "DeckOfPlayingCards", targets: ["DeckOfPlayingCards"]), - ], + name: "Example", + defaultLocalization: "en", + products: [], dependencies: [ - .package(url: "https://github.com/apple/example-package-fisheryates.git", from: "2.0.0"), - .package(url: "https://github.com/apple/example-package-playingcard.git", from: "3.0.0"), - ], - targets: [ - .target( - name: "DeckOfPlayingCards", - dependencies: ["FisherYates", "PlayingCard"]), - .testTarget( - name: "DeckOfPlayingCardsTests", - dependencies: ["DeckOfPlayingCards"]), + .package(name: "grpc-swift", url: "https://github.com/grpc/grpc-swift.git", from: "1.0.0"), ] ) ``` -When analysis is performed (e.g. `fossa analyze -o`), we will identify following as direct dependencies: -- https://github.com/apple/example-package-fisheryates.git -- https://github.com/apple/example-package-playingcard.git +We can update and resolve dependencies by performing `swift package update`. Executing this will create Package.resolved in the directory. An example file is shown below: + +```json +{ + "object": { + "pins": [ + { + "package": "grpc-swift", + "repositoryURL": "https://github.com/grpc/grpc-swift.git", + "state": { + "branch": null, + "revision": "14e1ea3350892a864386517c037e11fb68baf818", + "version": "1.3.0" + } + }, + { + "package": "swift-log", + "repositoryURL": "https://github.com/apple/swift-log.git", + "state": { + "branch": null, + "revision": "5d66f7ba25daf4f94100e7022febf3c75e37a6c7", + "version": "1.4.2" + } + } + ] + }, + "version": 1 +} + +``` +Note: Only a few pins are shown above for brevity. + +### `Package.swift` and `Package.resolved` + +When the analysis is performed (e.g. `fossa analyze -o`), we will identify the following as direct dependencies: + +- https://github.com/grpc/grpc-swift.git@1.3.0 + +If `Package.resolved` is discovered, the following deep dependencies will be identified, however, we will not identify the edges in the dependency graph: + +- https://github.com/apple/swift-log.git@1.4.2 + +If `Package.resolved` is not discovered, only direct dependencies will be reported. + +### Xcode Project and `Package.resolved` + +For Xcode project using swift package manager to manage swift package dependencies, Xcode project file named `project.pbxproj` will be analyzed. In the Xcode project file, `XCRemoteSwiftPackageReference` objects will be used to identify swift packages that are direct dependencies. For the analysis, at least one such reference must exist in the file. If no such references are found, we will not consider the Xcode project in the swift analysis. + +Excerpt from example `project.pbxproj`: + +``` +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 52; + objects = { + + ... + + 170A463726ECEDEF002DDFB8 /* XCRemoteSwiftPackageReference "example-package-deckofplayingcards" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/apple/example-package-deckofplayingcards"; + requirement = { + branch = main; + kind = branch; + }; + }; + + ... + }; + rootObject = 17874CD926C46B8500D16CA8 /* Project object */; +} +``` + +If the `Package.resolved` is discovered, deep dependencies will be identified. If not, only direct dependencies listed in xcode project file will be identified. In either case, no edges among dependencies will be reported. ## F.A.Q ### How do I *only perform analysis* for swift package dependencies? -You can explicitly specify analysis an target in `.fossa.yml` file. +You can explicitly specify the analysis target in `.fossa.yml` file. -Example below, will exclude all analysis targets except swift. +The example below will exclude all analysis targets except swift. ```yaml # .fossa.yml @@ -66,8 +137,32 @@ targets: - type: swift ``` +### Swift packages sourced from local directories are not discovered in the analysis. Is there a workaround? + +This is a current limitation. For swift package manager analysis, we only support non-path dependencies at the moment. +To include local dependencies, you can use `fossa-deps.yml` file to upload the local package for license scanning and analysis. + +```yaml +# in fossa-deps.yml + +vendored-dependencies: +- name: MyLocalPackage + path: /Jenkins/App/Resources/MyLocalPackage # path can be either a file or a folder. + version: 3.4.16 # revision will be set to the MD5 hash of the file path if left unspecified. +``` + +Note: License scanning currently operates by uploading the files at the specified path to a secure S3 bucket. All files that do not contain licenses are then removed after 2 weeks. +Refer to [User guide](../../userguide.md) for more details. + +### When performing `fossa list-targets`, Xcode project using swift packages are not getting discovered. + +For swift, we consider the Xcode project to be a valid Xcode project, if and only if it meets the following requirements: +- Xcode project file named: `project.pbxproj` exists in the directory. +- Xcode project file must be in ASCII plist format with UTF-8 encoding. +- Xcode project file has at least one object, with isa of `XCRemoteSwiftPackageReference`. + ## References - [Swift Package Manager](https://github.com/apple/swift-package-manager) - [Package.swift, must begin with version specifier](https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#about-the-swift-tools-version) -- [Package.swift, must be defined in single nested statement, and should not be modified after initialization](https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#package) \ No newline at end of file +- [Package.swift, must be defined in a single nested statement, and should not be modified after initialization](https://github.com/apple/swift-package-manager/blob/main/Documentation/PackageDescription.md#package) \ No newline at end of file diff --git a/spectrometer.cabal b/spectrometer.cabal index 711c5044e..ac746c1a3 100644 --- a/spectrometer.cabal +++ b/spectrometer.cabal @@ -99,6 +99,7 @@ common deps , prettyprinter >=1.6 && <1.8 , prettyprinter-ansi-terminal ^>=1.1.1 , random ^>=1.2.0 + , raw-strings-qq ^>=1.1 , req ^>=3.9.1 , retry ^>=0.9.0.0 , semver ^>=0.4.0.1 @@ -296,7 +297,10 @@ library Strategy.Ruby.GemfileLock Strategy.Scala Strategy.SwiftPM + Strategy.Swift.PackageResolved Strategy.Swift.PackageSwift + Strategy.Swift.Xcode.Pbxproj + Strategy.Swift.Xcode.PbxprojParser Strategy.Yarn Strategy.Yarn.V1.YarnLock Strategy.Yarn.V2.Lockfile @@ -393,7 +397,10 @@ test-suite unit-tests RPM.SpecFileSpec Ruby.BundleShowSpec Ruby.GemfileLockSpec + Swift.PackageResolvedSpec Swift.PackageSwiftSpec + Swift.Xcode.PbxprojSpec + Swift.Xcode.PbxprojParserSpec Yarn.V2.LockfileSpec Yarn.V2.ResolversSpec Yarn.YarnLockV1Spec diff --git a/src/App/Fossa/Analyze/Project.hs b/src/App/Fossa/Analyze/Project.hs index 456b23040..91208e0c3 100644 --- a/src/App/Fossa/Analyze/Project.hs +++ b/src/App/Fossa/Analyze/Project.hs @@ -22,7 +22,7 @@ mkResult basedir project dependencyResults = -- their dependencies would be filtered out. The real fix to this is to -- have a separate designation for "reachable" vs "direct" on nodes in a -- Graphing, where direct deps are inherently reachable. - if null (Graphing.directList graph) + if null (Graphing.directList graph) || shouldKeepUnreachableDeps (projectType project) then graph else Graphing.pruneUnreachable graph , projectResultGraphBreadth = dependencyGraphBreadth dependencyResults @@ -39,3 +39,7 @@ data ProjectResult = ProjectResult , projectResultGraphBreadth :: GraphBreadth , projectResultManifestFiles :: [SomeBase File] } + +shouldKeepUnreachableDeps :: Text -> Bool +shouldKeepUnreachableDeps "swift" = True +shouldKeepUnreachableDeps _ = False diff --git a/src/Strategy/Swift/PackageResolved.hs b/src/Strategy/Swift/PackageResolved.hs new file mode 100644 index 000000000..6535e82cd --- /dev/null +++ b/src/Strategy/Swift/PackageResolved.hs @@ -0,0 +1,78 @@ +module Strategy.Swift.PackageResolved ( + SwiftPackageResolvedFile (..), + SwiftResolvedPackage (..), + resolvedDependenciesOf, +) where + +import Data.Aeson ( + FromJSON (parseJSON), + Object, + withObject, + (.:), + (.:?), + ) +import Data.Aeson.Types (Parser) +import Data.Foldable (asum) +import Data.Text (Text) +import DepTypes (DepType (GitType), Dependency (..), VerConstraint (CEq)) + +data SwiftPackageResolvedFile = SwiftPackageResolvedFile + { version :: Integer + , pinnedPackages :: [SwiftResolvedPackage] + } + deriving (Show, Eq, Ord) + +data SwiftResolvedPackage = SwiftResolvedPackage + { package :: Text + , repositoryURL :: Text + , repositoryBranch :: Maybe Text + , repositoryRevision :: Maybe Text + , repositoryVersion :: Maybe Text + } + deriving (Show, Eq, Ord) + +instance FromJSON SwiftPackageResolvedFile where + parseJSON = withObject "Package.resolved content" $ \obj -> + SwiftPackageResolvedFile <$> obj .: "version" + <*> (obj .: "object" |> "pins") + +(|>) :: FromJSON a => Parser Object -> Text -> Parser a +(|>) parser key = do + obj <- parser + obj .: key + +(|?>) :: FromJSON a => Parser (Maybe Object) -> Text -> Parser (Maybe a) +(|?>) parser key = do + obj <- parser + case obj of + Nothing -> pure Nothing + Just o -> o .:? key + +instance FromJSON SwiftResolvedPackage where + parseJSON = withObject "Package.resolved pinned object" $ \obj -> + SwiftResolvedPackage <$> obj .: "package" + <*> obj .: "repositoryURL" + <*> (obj .:? "state" |?> "branch") + <*> (obj .:? "state" |?> "revision") + <*> (obj .:? "state" |?> "version") + +-- Note, Package.resolved does not include path dependencies. +resolvedDependenciesOf :: SwiftPackageResolvedFile -> [Dependency] +resolvedDependenciesOf resolvedContent = map toDependency $ pinnedPackages resolvedContent + where + toDependency :: SwiftResolvedPackage -> Dependency + toDependency pkg = + Dependency + { dependencyType = GitType + , dependencyName = repositoryURL pkg + , dependencyVersion = + CEq + <$> asum + [ repositoryRevision pkg + , repositoryVersion pkg + , repositoryBranch pkg + ] + , dependencyLocations = [] + , dependencyEnvironments = [] + , dependencyTags = mempty + } diff --git a/src/Strategy/Swift/PackageSwift.hs b/src/Strategy/Swift/PackageSwift.hs index e1a9d8351..d4f11a04a 100644 --- a/src/Strategy/Swift/PackageSwift.hs +++ b/src/Strategy/Swift/PackageSwift.hs @@ -1,13 +1,15 @@ module Strategy.Swift.PackageSwift ( analyzePackageSwift, + SwiftPackageGitDep (..), + SwiftPackageGitDepRequirement (..), + toConstraint, + isGitRefConstraint, -- * for testing, buildGraph, parsePackageSwiftFile, SwiftPackage (..), SwiftPackageDep (..), - SwiftPackageGitDep (..), - SwiftPackageGitDepRequirement (..), ) where import Control.Applicative (Alternative ((<|>)), optional) @@ -15,12 +17,14 @@ import Control.Effect.Diagnostics (Diagnostics, context) import Control.Monad (void) import Data.Foldable (asum) import Data.Map.Strict qualified as Map +import Data.Set (Set, fromList, member) import Data.Text (Text) import Data.Void (Void) import DepTypes (DepType (GitType, SwiftType), Dependency (..), VerConstraint (CEq)) -import Effect.ReadFS (Has, ReadFS, readContentsParser) -import Graphing (Graphing, directs, induceJust) +import Effect.ReadFS (Has, ReadFS, readContentsJson, readContentsParser) +import Graphing (Graphing, deeps, directs, induceJust, promoteToDirect) import Path +import Strategy.Swift.PackageResolved (SwiftPackageResolvedFile, resolvedDependenciesOf) import Text.Megaparsec ( MonadParsec (takeWhile1P, try), Parsec, @@ -201,20 +205,38 @@ parsePackageSwiftFile = do -- | Analysis -- * -analyzePackageSwift :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> m (Graphing.Graphing Dependency) -analyzePackageSwift manifestFile = - do - manifestContent <- - context - "Identifying dependencies in Package.swift" - $ readContentsParser parsePackageSwiftFile manifestFile - context "Building dependency graph" $ - pure $ buildGraph manifestContent +analyzePackageSwift :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> Maybe (Path Abs File) -> m (Graphing.Graphing Dependency) +analyzePackageSwift manifestFile resolvedFile = do + manifestContent <- context "Identifying dependencies in Package.swift" $ readContentsParser parsePackageSwiftFile manifestFile + + packageResolvedContent <- case resolvedFile of + Nothing -> pure Nothing + Just packageResolved -> context "Identifying dependencies in Package.resolved" $ readContentsJson packageResolved + + context "Building dependency graph" $ pure $ buildGraph manifestContent packageResolvedContent -- | Graph Building -- * -buildGraph :: SwiftPackage -> Graphing.Graphing Dependency -buildGraph pkg = induceJust $ directs (map toDependency $ packageDependencies pkg) +buildGraph :: SwiftPackage -> Maybe SwiftPackageResolvedFile -> Graphing.Graphing Dependency +buildGraph manifestContent maybeResolvedContent = + case maybeResolvedContent of + Nothing -> induceJust $ directs (map toDependency $ packageDependencies manifestContent) + -- If dependency (url) is present in the manifest, promote them to direct dependency + -- Otherwise, keep them as deep dependencies. Since Package.resolved does not include + -- dependencies sourced from local path, we do not need to do any filtering. + Just resolvedContent -> + promoteToDirect (isDirect depInManifest) $ + deeps $ resolvedDependenciesOf resolvedContent + where + isDirect :: Set Text -> Dependency -> Bool + isDirect s dep = (dependencyName dep) `member` s + + depInManifest :: Set Text + depInManifest = fromList $ map getName $ packageDependencies manifestContent + + getName :: SwiftPackageDep -> Text + getName (PathSource path) = path + getName (GitSource pkg) = srcOf pkg toDependency :: SwiftPackageDep -> Maybe Dependency toDependency (PathSource _) = Nothing diff --git a/src/Strategy/Swift/Xcode/Pbxproj.hs b/src/Strategy/Swift/Xcode/Pbxproj.hs new file mode 100644 index 000000000..3d209f982 --- /dev/null +++ b/src/Strategy/Swift/Xcode/Pbxproj.hs @@ -0,0 +1,115 @@ +module Strategy.Swift.Xcode.Pbxproj ( + analyzeXcodeProjForSwiftPkg, + hasSomeSwiftDeps, + + -- * for testing + buildGraph, + XCRemoteSwiftPackageReference (..), + swiftPackageReferencesOf, +) where + +import Control.Effect.Diagnostics (Diagnostics, context) +import Data.Map (Map) +import Data.Map.Strict qualified as Map +import Data.Maybe (mapMaybe) +import Data.Set (fromList, member) +import Data.Text (Text) +import DepTypes (DepType (GitType, SwiftType), Dependency (..)) +import Effect.ReadFS (Has, ReadFS, readContentsJson, readContentsParser) +import Graphing (Graphing, deeps, directs, promoteToDirect) +import Path +import Strategy.Swift.PackageResolved (SwiftPackageResolvedFile, resolvedDependenciesOf) +import Strategy.Swift.PackageSwift ( + SwiftPackageGitDepRequirement (..), + isGitRefConstraint, + toConstraint, + ) +import Strategy.Swift.Xcode.PbxprojParser (AsciiValue (..), PbxProj (..), lookupText, objectsFromIsa, parsePbxProj, textOf) + +-- | Represents the version rules for a Swift Package as defined in Xcode project file. +data XCRemoteSwiftPackageReference = XCRemoteSwiftPackageReference + { -- | Represents repositoryURL field from project file. + urlOf :: Text + , -- | Represents requirement field from project file. + requirementOf :: SwiftPackageGitDepRequirement + } + deriving (Show, Eq, Ord) + +swiftPackageReferencesOf :: PbxProj -> [XCRemoteSwiftPackageReference] +swiftPackageReferencesOf pbx = mapMaybe toSwiftPkgRef swiftPkgRefObjects + where + swiftPkgRefObjects :: [Map Text AsciiValue] + swiftPkgRefObjects = maybe [] (objectsFromIsa "XCRemoteSwiftPackageReference") (objects pbx) + + toSwiftPkgRef :: Map Text AsciiValue -> Maybe XCRemoteSwiftPackageReference + toSwiftPkgRef candidate = case (repositoryURL candidate, requirement candidate) of + (Just url, Just req) -> Just $ XCRemoteSwiftPackageReference url req + (_, _) -> Nothing + + repositoryURL :: Map Text AsciiValue -> Maybe Text + repositoryURL v = Map.lookup "repositoryURL" v >>= textOf + + requirement :: Map Text AsciiValue -> Maybe SwiftPackageGitDepRequirement + requirement v = Map.lookup "requirement" v >>= toReferenceRequirement + + toReferenceRequirement :: AsciiValue -> Maybe SwiftPackageGitDepRequirement + toReferenceRequirement value = + case kind of + Just "upToNextMajorVersion" -> UpToNextMajor <$> get "minimumVersion" + Just "upToNextMinorVersion" -> UpToNextMinor <$> get "minimumVersion" + Just "versionRange" -> ClosedInterval <$> ((,) <$> get "minimumVersion" <*> get "maximumVersion") + Just "branch" -> Branch <$> get "branch" + Just "revision" -> Revision <$> get "revision" + Just "exactVersion" -> Exact <$> get "version" + Just _ -> Nothing + Nothing -> Nothing + where + get = lookupText value + kind = get "kind" + +toDependency :: XCRemoteSwiftPackageReference -> Dependency +toDependency src = + Dependency + { dependencyType = depType + , dependencyName = urlOf src + , dependencyVersion = Just $ toConstraint $ requirementOf src + , dependencyLocations = [] + , dependencyEnvironments = [] + , dependencyTags = Map.empty + } + where + depType :: DepType + depType = + if isGitRefConstraint $ requirementOf src + then GitType + else SwiftType + +buildGraph :: PbxProj -> Maybe SwiftPackageResolvedFile -> Graphing.Graphing Dependency +buildGraph projFile maybeResolvedContent = + case maybeResolvedContent of + Nothing -> directs $ map toDependency $ swiftPackageReferencesOf projFile + Just resolvedContent -> promoteToDirect isDirect $ deeps $ resolvedDependenciesOf resolvedContent + where + isDirect :: Dependency -> Bool + isDirect dep = (dependencyName dep) `member` fromList (map urlOf $ swiftPackageReferencesOf projFile) + +-- | Checks if XCode Project File has at-least one swift dependency. +-- It does by counting instances of `XCRemoteSwiftPackageReference` in the project file. +hasSomeSwiftDeps :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> m Bool +hasSomeSwiftDeps projFile = do + xCodeProjContent <- readContentsParser parsePbxProj projFile + pure $ (not . null) (swiftPackageReferencesOf xCodeProjContent) + +analyzeXcodeProjForSwiftPkg :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs File -> Maybe (Path Abs File) -> m (Graphing.Graphing Dependency) +analyzeXcodeProjForSwiftPkg xcodeProjFile resolvedFile = do + xCodeProjContent <- + context "Identifying swift package references in xcode project file" $ + readContentsParser parsePbxProj xcodeProjFile + + packageResolvedContent <- case resolvedFile of + Nothing -> pure Nothing + Just packageResolved -> + context "Identifying dependencies in Package.resolved" $ + readContentsJson packageResolved + + context "Building dependency graph" $ pure $ buildGraph xCodeProjContent packageResolvedContent diff --git a/src/Strategy/Swift/Xcode/PbxprojParser.hs b/src/Strategy/Swift/Xcode/PbxprojParser.hs new file mode 100644 index 000000000..57c8b2ea0 --- /dev/null +++ b/src/Strategy/Swift/Xcode/PbxprojParser.hs @@ -0,0 +1,187 @@ +-- | Module : Strategy.Xcode.PbxprojParser +-- +-- Provides elementary parsing of xcode's pbxproj.project file. +-- Xcode uses plist ascii, encoded in UTF-8 to perform record configurations. +-- +-- There is no official spec, for the file format. +-- +-- It can represents data in: +-- * Binary +-- * Date +-- * String +-- * Number +-- * List +-- * Dictionary +-- +-- Relevant References: +-- * For ASCII types: https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/PropertyLists/OldStylePlists/OldStylePLists.html +-- * Unofficial References: +-- * http://www.monobjc.net/xcode-project-file-format.html +-- +-- We intentionally parse all types into one of String, List, and Dictionary. +-- We do not distinguish between types of Xcode specific configurations. +module Strategy.Swift.Xcode.PbxprojParser ( + parsePbxProj, + PbxProj (..), + AsciiValue (..), + objectsFromIsa, + lookupText, + textOf, + lookupTextFromAsciiDict, + + -- * for testing only + parseAsciiText, + parseAsciiList, + parseAsciiDict, + parseAsciiValue, +) where + +import Data.Functor (void) +import Data.Map (Map) +import Data.Map.Strict qualified as Map +import Data.Maybe (mapMaybe) +import Data.String.Conversion (ToText (toText)) +import Data.Text (Text) +import Data.Void (Void) +import Text.Megaparsec ( + MonadParsec (takeWhile1P, try), + Parsec, + between, + many, + noneOf, + sepEndBy, + some, + (), + (<|>), + ) +import Text.Megaparsec.Char (char, string) +import Text.Megaparsec.Char.Lexer qualified as Lexer + +type Parser = Parsec Void Text + +sc :: Parser () +sc = + Lexer.space + (void $ some $ char ' ' <|> char '\t' <|> char '\n' <|> char '\r') + (Lexer.skipLineComment "//") + (Lexer.skipBlockComment "/*" "*/") + +lexeme :: Parser a -> Parser a +lexeme = Lexer.lexeme sc + +symbol :: Text -> Parser Text +symbol = Lexer.symbol sc + +betweenCurlyBrackets :: Parser a -> Parser a +betweenCurlyBrackets = between (symbol "{") (symbol "}") + +betweenParentheses :: Parser a -> Parser a +betweenParentheses = between (symbol "(") (symbol ")") + +parseQuotedText :: Parser Text +parseQuotedText = between (symbol "\"") (symbol "\"") quoted + where + quoted :: Parser Text + quoted = toText <$> many (nullifiedQuote <|> notEscapedQuote) + + nullifiedQuote :: Parser Char + nullifiedQuote = string "\\\"" >> pure '"' + + notEscapedQuote :: Parser Char + notEscapedQuote = noneOf ['\"'] + +parseText :: Parser Text +parseText = takeWhile1P (Just "text") (\c -> c `notElem` [';', ',', ')', ' ', '\t', '\n', '\r']) + +-- | Potential type represented in Ascii plist file. +-- Reference : https://developer.apple.com/library/archive/documentation/Cocoa/Conceptual/PropertyLists/OldStylePlists/OldStylePLists.html +data AsciiValue + = -- | Represents SomeText or "SomeText" + -- Since we are only interested in textual representation of package name, and package version + -- We represent potential binary, date, boolean ascii type as text. + AText Text + | -- | Represents {key = value;} + ADict (Map Text AsciiValue) + | -- | Represents (A, B,) + AList [AsciiValue] + deriving (Show, Eq, Ord) + +data AsciiKeyValue = AsciiKeyValue Text AsciiValue deriving (Show, Eq, Ord) + +parseAsciiText :: Parser AsciiValue +parseAsciiText = AText <$> lexeme (try parseQuotedText <|> parseText) + +parseAsciiList :: Parser AsciiValue +parseAsciiList = AList <$> betweenParentheses (sepEndBy parseAsciiValue (symbol ",")) + +parseAsciiValue :: Parser AsciiValue +parseAsciiValue = try parseAsciiDict <|> try parseAsciiList <|> parseAsciiText + +parseAsciiDict :: Parser AsciiValue +parseAsciiDict = ADict <$> (Map.fromList <$> lexeme (betweenCurlyBrackets $ sepEndBy (try parseAsciiKeyValue) (symbol ";"))) + +parseAsciiKeyValue :: Parser (Text, AsciiValue) +parseAsciiKeyValue = do + key <- lexeme parseText <* symbol "=" + value <- lexeme $ try parseAsciiList <|> try parseAsciiDict <|> parseAsciiText + pure (key, value) + +-- | Represents Xcode's pbxproj.project file elementary structure. +-- Reference: http://www.monobjc.net/xcode-project-file-format.html +data PbxProj = PbxProj + { archiveVersion :: Text + , objectVersion :: Text + , rootObject :: Text + , classes :: Maybe AsciiValue + , objects :: Maybe AsciiValue + } + deriving (Show, Eq, Ord) + +lookupTextFromAsciiDict :: AsciiValue -> Text -> Maybe AsciiValue +lookupTextFromAsciiDict (AText _) _ = Nothing +lookupTextFromAsciiDict (AList _) _ = Nothing +lookupTextFromAsciiDict (ADict val) key = Map.lookup key val + +textOf :: AsciiValue -> Maybe Text +textOf (AText t) = Just t +textOf _ = Nothing + +lookupText :: AsciiValue -> Text -> Maybe Text +lookupText v key = (v `lookupTextFromAsciiDict` key) >>= textOf + +supportedEncoding :: Text +supportedEncoding = "UTF8" + +parsePbxProj :: Parser PbxProj +parsePbxProj = do + _ <- symbol ("// !$*" <> supportedEncoding <> "*$!") "to have UTF8 Encoding!" + allValues <- parseAsciiDict + + archiveVersion <- case (textOf =<< (allValues `lookupTextFromAsciiDict` "archiveVersion")) of + Nothing -> fail "could not find archiveVersion" + Just av -> pure av + + objectVersion <- case (textOf =<< (allValues `lookupTextFromAsciiDict` "objectVersion")) of + Nothing -> fail "could not find objectVersion" + Just ov -> pure ov + + rootObject <- case (textOf =<< (allValues `lookupTextFromAsciiDict` "rootObject")) of + Nothing -> fail "could not find rootObject" + Just ro -> pure ro + + let classes = (allValues `lookupTextFromAsciiDict` "classes") + let objects = (allValues `lookupTextFromAsciiDict` "objects") + pure $ PbxProj archiveVersion objectVersion rootObject classes objects + +-- | Gets list of objects with given isa value. +objectsFromIsa :: Text -> AsciiValue -> [Map Text AsciiValue] +objectsFromIsa _ (AText _) = [] +objectsFromIsa _ (AList _) = [] +objectsFromIsa key (ADict val) = mapMaybe getDict $ Map.elems filteredMap + where + filteredMap :: Map Text AsciiValue + filteredMap = Map.filterWithKey (\_ v -> Just key == (textOf =<< v `lookupTextFromAsciiDict` "isa")) val + + getDict :: AsciiValue -> Maybe (Map Text AsciiValue) + getDict (ADict v) = Just v + getDict _ = Nothing diff --git a/src/Strategy/SwiftPM.hs b/src/Strategy/SwiftPM.hs index ab4bf64e1..5bd9ed111 100644 --- a/src/Strategy/SwiftPM.hs +++ b/src/Strategy/SwiftPM.hs @@ -1,55 +1,129 @@ +{-# LANGUAGE QuasiQuotes #-} + module Strategy.SwiftPM ( discover, - findProjects, mkProject, ) where import Control.Carrier.Simple (Has) import Control.Effect.Diagnostics (Diagnostics, context) +import Data.Functor (($>)) +import Data.Maybe (listToMaybe) import Discovery.Walk ( - WalkStep (WalkContinue), + WalkStep (WalkContinue, WalkSkipSome), findFileNamed, walk', ) +import Effect.Logger (Logger (..), Pretty (pretty), logDebug) import Effect.ReadFS (ReadFS) import Path import Strategy.Swift.PackageSwift (analyzePackageSwift) +import Strategy.Swift.Xcode.Pbxproj (analyzeXcodeProjForSwiftPkg, hasSomeSwiftDeps) import Types (DependencyResults (..), DiscoveredProject (..), GraphBreadth (..)) +data SwiftProject + = PackageProject SwiftPackageProject + | XcodeProject XcodeProjectUsingSwiftPm + deriving (Show, Eq, Ord) + data SwiftPackageProject = SwiftPackageProject - { manifest :: Path Abs File - , projectDir :: Path Abs Dir - , resolved :: Maybe (Path Abs File) + { swiftPkgManifest :: Path Abs File + , swiftPkgProjectDir :: Path Abs Dir + , swiftPkgResolved :: Maybe (Path Abs File) + } + deriving (Show, Eq, Ord) + +data XcodeProjectUsingSwiftPm = XcodeProjectUsingSwiftPm + { xCodeProjectFile :: Path Abs File + , xCodeProjectDir :: Path Abs Dir + , xCodeResolvedFile :: Maybe (Path Abs File) } deriving (Show, Eq, Ord) -discover :: (Has ReadFS sig m, Has Diagnostics sig m, Has ReadFS rsig run, Has Diagnostics rsig run) => Path Abs Dir -> m [DiscoveredProject run] +discover :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m, Has ReadFS rsig run, Has Diagnostics rsig run) => Path Abs Dir -> m [DiscoveredProject run] discover dir = context "Swift" $ do - projects <- context "Finding projects" $ findProjects dir - pure (map mkProject projects) + swiftPackageProjects <- context "Finding swift package projects" $ findSwiftPackageProjects dir + xCodeProjects <- context "Finding xcode projects using swift package manager" $ findXcodeProjects dir + pure $ map mkProject (swiftPackageProjects ++ xCodeProjects) + +findSwiftPackageProjects :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m [SwiftProject] +findSwiftPackageProjects = walk' $ \dir _ files -> do + let packageManifestFile = findFileNamed "Package.swift" files + let packageResolvedFile = findFileNamed "Package.resolved" files + case (packageManifestFile, packageResolvedFile) of + -- If the Package.swift exists, than it is swift package project. + -- Use Package.swift as primary source of truth. + (Just manifestFile, resolvedFile) -> pure ([PackageProject $ SwiftPackageProject manifestFile dir resolvedFile], WalkSkipSome [".build"]) + -- Package.resolved without Package.swift or Xcode project file is not a valid swift project. + (Nothing, _) -> pure ([], WalkContinue) -findProjects :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m [SwiftPackageProject] -findProjects = walk' $ \dir _ files -> do - case findFileNamed "Package.swift" files of +findXcodeProjects :: (Has ReadFS sig m, Has Diagnostics sig m, Has Logger sig m) => Path Abs Dir -> m [SwiftProject] +findXcodeProjects = walk' $ \dir _ files -> do + let xcodeProjectFile = findFileNamed "project.pbxproj" files + case xcodeProjectFile of Nothing -> pure ([], WalkContinue) - Just file -> pure ([SwiftPackageProject file dir Nothing], WalkContinue) + Just projFile -> do + resolvedFile <- findFirstResolvedFileRecursively dir + xCodeProjWithDependencies <- hasSomeSwiftDeps projFile + if xCodeProjWithDependencies + then pure ([XcodeProject $ XcodeProjectUsingSwiftPm projFile dir resolvedFile], WalkSkipSome [".build"]) + else debugXCodeWithoutSwiftDeps projFile $> ([], WalkContinue) + +-- | Walks directory and finds first file named 'Package.resolved'. +-- XCode projects using swift package manager retain Package.resolved, +-- not in the same directory as project file, but rather in workspace's xcshareddata/swiftpm directory. +-- Reference: https://developer.apple.com/documentation/swift_packages/adding_package_dependencies_to_your_app. +findFirstResolvedFileRecursively :: (Has ReadFS sig m, Has Diagnostics sig m) => Path Abs Dir -> m (Maybe (Path Abs File)) +findFirstResolvedFileRecursively baseDir = listToMaybe <$> walk' findFile baseDir + where + isParentDirSwiftPm :: Path Abs Dir -> Bool + isParentDirSwiftPm d = (dirname d) == [reldir|swiftpm|] + + findFile :: forall f. Applicative f => Path Abs Dir -> [Path Abs Dir] -> [Path Abs File] -> f ([Path Abs File], WalkStep) + findFile dir _ files = do + let foundFile = findFileNamed "Package.resolved" files + case (foundFile) of + (Just ff) -> + if (isParentDirSwiftPm dir) + then pure ([ff], WalkSkipSome [".build"]) + else pure ([], WalkContinue) + _ -> pure ([], WalkContinue) + +debugXCodeWithoutSwiftDeps :: Has Logger sig m => Path Abs File -> m () +debugXCodeWithoutSwiftDeps projFile = + (logDebug . pretty) $ + "XCode project file (" + <> show projFile + <> "), did not have any XCRemoteSwiftPackageReference, ignoring from swift analyses." -mkProject :: (Has ReadFS sig n, Has Diagnostics sig n) => SwiftPackageProject -> DiscoveredProject n +mkProject :: (Has ReadFS sig n, Has Diagnostics sig n) => SwiftProject -> DiscoveredProject n mkProject project = DiscoveredProject { projectType = "swift" , projectBuildTargets = mempty , projectDependencyResults = const $ getDeps project - , projectPath = projectDir project + , projectPath = getProjectDir , projectLicenses = pure [] } + where + getProjectDir :: Path Abs Dir + getProjectDir = case project of + PackageProject prj -> swiftPkgProjectDir prj + XcodeProject prj -> xCodeProjectDir prj -getDeps :: (Has ReadFS sig m, Has Diagnostics sig m) => SwiftPackageProject -> m DependencyResults +getDeps :: (Has ReadFS sig m, Has Diagnostics sig m) => SwiftProject -> m DependencyResults getDeps project = do - graph <- analyzePackageSwift $ manifest project + graph <- case project of + PackageProject prj -> analyzePackageSwift (swiftPkgManifest prj) (swiftPkgResolved prj) + XcodeProject prj -> analyzeXcodeProjForSwiftPkg (xCodeProjectFile prj) (xCodeResolvedFile prj) pure $ DependencyResults { dependencyGraph = graph , dependencyGraphBreadth = Partial - , dependencyManifestFiles = [manifest project] + , dependencyManifestFiles = manifestFiles } + where + manifestFiles :: [Path Abs File] + manifestFiles = case project of + PackageProject prj -> [swiftPkgManifest prj] + XcodeProject prj -> [xCodeProjectFile prj] diff --git a/test/Swift/PackageResolvedSpec.hs b/test/Swift/PackageResolvedSpec.hs new file mode 100644 index 000000000..56731b8aa --- /dev/null +++ b/test/Swift/PackageResolvedSpec.hs @@ -0,0 +1,41 @@ +module Swift.PackageResolvedSpec ( + spec, +) where + +import Data.Aeson (decodeFileStrict') +import Strategy.Swift.PackageResolved ( + SwiftPackageResolvedFile (..), + SwiftResolvedPackage (..), + ) +import Test.Hspec (Spec, describe, it, shouldBe) + +expectedResolvedContent :: SwiftPackageResolvedFile +expectedResolvedContent = + SwiftPackageResolvedFile + 1 + [ SwiftResolvedPackage + "grpc-swift" + "https://github.com/grpc/grpc-swift.git" + Nothing + (Just "9e464a75079928366aa7041769a271fac89271bf") + (Just "1.0.0") + , SwiftResolvedPackage + "Opentracing" + "https://github.com/undefinedlabs/opentracing-objc" + (Just "master") + Nothing + Nothing + , SwiftResolvedPackage + "Reachability" + "https://github.com/ashleymills/Reachability.swift" + Nothing + Nothing + (Just "5.1.0") + ] + +spec :: Spec +spec = do + describe "parse Package.resolved file" $ + it "should parse content correctly" $ do + resolvedFile <- decodeFileStrict' "test/Swift/testdata/Package.resolved" + resolvedFile `shouldBe` Just expectedResolvedContent diff --git a/test/Swift/PackageSwiftSpec.hs b/test/Swift/PackageSwiftSpec.hs index 144442b72..608d6cea7 100644 --- a/test/Swift/PackageSwiftSpec.hs +++ b/test/Swift/PackageSwiftSpec.hs @@ -7,6 +7,7 @@ import Data.Text (Text) import Data.Text.IO qualified as TIO import DepTypes (DepType (GitType, SwiftType), Dependency (..), VerConstraint (CEq)) import GraphUtil (expectDeps, expectDirect, expectEdges) +import Strategy.Swift.PackageResolved (SwiftPackageResolvedFile (..), SwiftResolvedPackage (..)) import Strategy.Swift.PackageSwift ( SwiftPackage (..), SwiftPackageDep (..), @@ -82,7 +83,7 @@ spec = do Left failCode -> expectationFailure $ show failCode Right result -> result `shouldBe` expectedSwiftPackage - describe "buildGraph" $ do + describe "buildGraph, when no resolved content is discovered" $ do it "should use git dependency type, when constraint is of branch, revision, or exact type" $ do let expectedDeps = [ Dependency GitType "some-url" (CEq <$> Just "some-ref") [] [] Map.empty @@ -90,29 +91,34 @@ spec = do , Dependency GitType "some-url" (CEq <$> Just "1.0.0") [] [] Map.empty ] let graph = - buildGraph $ - SwiftPackage - "5.3" - [ GitSource $ gitDepWithRevision "some-url" "some-ref" - , GitSource $ gitDepWithBranch "some-url" "some-branch" - , GitSource $ gitDepExactly "some-url" "1.0.0" - ] + buildGraph + ( SwiftPackage + "5.3" + [ GitSource $ gitDepWithRevision "some-url" "some-ref" + , GitSource $ gitDepWithBranch "some-url" "some-branch" + , GitSource $ gitDepExactly "some-url" "1.0.0" + ] + ) + Nothing + expectDirect expectedDeps graph expectDeps expectedDeps graph expectEdges [] graph it "should use swift dependency type, when constraint uses follows, range, upToNextMajor, or upToNextMinor" $ do let graph = - buildGraph $ - SwiftPackage - "5.3" - [ GitSource $ gitDepWithoutConstraint "some-url-dep" - , GitSource $ gitDepFrom "some-url-dep" "3.0.0" - , GitSource $ gitDepUpToNextMajor "some-url-dep" "2.0.0" - , GitSource $ gitDepUpToNextMinor "some-url-dep" "1.0.0" - , GitSource $ gitDepWithRhsHalfOpenInterval "some-url-dep" "2.5.0" "2.5.6" - , GitSource $ gitDepWithClosedRange "some-url-dep" "3.0.5" "3.0.7" - ] + buildGraph + ( SwiftPackage + "5.3" + [ GitSource $ gitDepWithoutConstraint "some-url-dep" + , GitSource $ gitDepFrom "some-url-dep" "3.0.0" + , GitSource $ gitDepUpToNextMajor "some-url-dep" "2.0.0" + , GitSource $ gitDepUpToNextMinor "some-url-dep" "1.0.0" + , GitSource $ gitDepWithRhsHalfOpenInterval "some-url-dep" "2.5.0" "2.5.6" + , GitSource $ gitDepWithClosedRange "some-url-dep" "3.0.5" "3.0.7" + ] + ) + Nothing let expectedDeps = [ Dependency SwiftType "some-url-dep" Nothing [] [] Map.empty , Dependency SwiftType "some-url-dep" (CEq <$> Just "^3.0.0") [] [] Map.empty @@ -124,3 +130,48 @@ spec = do expectDirect expectedDeps graph expectDeps expectedDeps graph expectEdges [] graph + + describe "buildGraph, when resolved content is discovered" $ do + it "should use git dependency type, when constraint is of branch, revision, or exact type" $ do + let expectedDirectDeps = + [ Dependency GitType "dep-A" (CEq <$> Just "some-rev-A") [] [] Map.empty + , Dependency GitType "dep-B" (CEq <$> Just "some-rev-B") [] [] Map.empty + ] + let expectedDeepDeps = [Dependency GitType "dep-A-C" (CEq <$> Just "5.1.0") [] [] Map.empty] + + let graph = + buildGraph + ( SwiftPackage "5.3" $ + map + GitSource + [ gitDepFrom "dep-A" "1.2.2" + , gitDepFrom "dep-B" "1.2.2" + ] + ) + ( Just $ + SwiftPackageResolvedFile + 1 + [ SwiftResolvedPackage + "depA" + "dep-A" + Nothing + (Just "some-rev-A") + (Just "1.2.5") + , SwiftResolvedPackage + "depB" + "dep-B" + Nothing + (Just "some-rev-B") + (Just "1.2.6") + , SwiftResolvedPackage + "depAC" + "dep-A-C" + Nothing + Nothing + (Just "5.1.0") + ] + ) + + expectDirect expectedDirectDeps graph + expectDeps (expectedDirectDeps ++ expectedDeepDeps) graph + expectEdges [] graph diff --git a/test/Swift/Xcode/PbxprojParserSpec.hs b/test/Swift/Xcode/PbxprojParserSpec.hs new file mode 100644 index 000000000..9bed52c81 --- /dev/null +++ b/test/Swift/Xcode/PbxprojParserSpec.hs @@ -0,0 +1,193 @@ +{-# LANGUAGE QuasiQuotes #-} + +module Swift.Xcode.PbxprojParserSpec ( + spec, +) where + +import Data.Map.Strict qualified as Map +import Data.Text (Text) +import Data.Text.IO qualified as TIO +import Data.Void (Void) +import Strategy.Swift.Xcode.PbxprojParser ( + AsciiValue (..), + PbxProj (..), + parseAsciiDict, + parseAsciiList, + parseAsciiText, + parseAsciiValue, + parsePbxProj, + ) +import Test.Hspec ( + Expectation, + Spec, + describe, + expectationFailure, + it, + runIO, + shouldBe, + shouldContain, + shouldNotBe, + ) +import Test.Hspec.Megaparsec (shouldParse) +import Text.Megaparsec ( + Parsec, + errorBundlePretty, + parse, + runParser, + ) +import Text.RawString.QQ (r) + +parseMatch :: (Show a, Eq a) => Parsec Void Text a -> Text -> a -> Expectation +parseMatch parser input expected = parse parser "" input `shouldParse` expected + +simplePbxProjFile :: Text +simplePbxProjFile = + [r|// !$*UTF8*$! +{ + // some line comment + archiveVersion = 1; + classes = { + }; + objectVersion = 52; + objects = { + /* Begin PBXBuildFile section */ + 172D94BF26C5D824008A4DB2 /* Vapor in Frameworks */ = {isa = PBXBuildFile; productRef = 172D94BE26C5D824008A4DB2 /* Vapor */; }; + }; + rootObject = 17874CD926C46B8500D16CA9 /* Project object */; +}|] + +unSupportedPbxProjFile :: Text +unSupportedPbxProjFile = + [r|// !$*NOT-UTF8*$! +{ + archiveVersion = 1; +}|] + +spec :: Spec +spec = do + describe "parseAsciiText" $ do + let shouldParseInto = parseMatch parseAsciiText + + it "should parse text" $ do + "a" `shouldParseInto` AText "a" + "ab" `shouldParseInto` AText "ab" + "ab-c" `shouldParseInto` AText "ab-c" + "ab-c.d" `shouldParseInto` AText "ab-c.d" + + it "should parse quoted text" $ do + [r|"ab-c.d e"|] `shouldParseInto` AText [r|ab-c.d e|] + [r|"\"$(A)/$(B)\""|] `shouldParseInto` AText [r|"$(A)/$(B)"|] + [r|"$(A)\..\A\B"|] `shouldParseInto` AText [r|$(A)\..\A\B|] + [r|"exp A=\"${B:=0}\"\necho \"exp C=${D}\" > \"${E}/../.F.env\"\n if [-z \"${Z}\"]; \n"|] + `shouldParseInto` AText [r|exp A="${B:=0}"\necho "exp C=${D}" > "${E}/../.F.env"\n if [-z "${Z}"]; \n|] + + describe "parseAsciiList" $ do + let shouldParseInto = parseMatch parseAsciiList + it "should parse empty list" $ do + "( )" `shouldParseInto` AList [] + "()" `shouldParseInto` AList [] + + it "should parse list of text" $ do + "( a )" `shouldParseInto` AList [AText "a"] + "( a, )" `shouldParseInto` AList [AText "a"] + "( a, b )" `shouldParseInto` AList [AText "a", AText "b"] + "( a, b, )" `shouldParseInto` AList [AText "a", AText "b"] + "(\na,\nb,\n)" `shouldParseInto` AList [AText "a", AText "b"] + + it "should parse list of dictionary types" $ + "( { b = c } )" `shouldParseInto` AList [ADict $ Map.fromList [("b", AText "c")]] + + it "should parse list of mixed types" $ do + "( a, { b = c } )" `shouldParseInto` AList [AText "a", ADict $ Map.fromList [("b", AText "c")]] + "( a, { b = c }, )" `shouldParseInto` AList [AText "a", ADict $ Map.fromList [("b", AText "c")]] + "(\na,\n{ b = c }\n)" `shouldParseInto` AList [AText "a", ADict $ Map.fromList [("b", AText "c")]] + + describe "parseAsciiDict" $ do + let shouldParseInto = parseMatch parseAsciiDict + it "should parse empty dictionary" $ do + "{ }" `shouldParseInto` ADict (Map.empty) + "{}" `shouldParseInto` ADict (Map.empty) + + it "should parse dictionary with key, and value of text" $ do + "{ b = c }" `shouldParseInto` ADict (Map.fromList [("b", AText "c")]) + "{ b = c; }" `shouldParseInto` ADict (Map.fromList [("b", AText "c")]) + "{ b = c; d = \"e\" }" `shouldParseInto` ADict (Map.fromList [("b", AText "c"), ("d", AText "e")]) + + it "should parse dictionary with key, and value of list" $ do + "{ f = () }" `shouldParseInto` ADict (Map.fromList [("f", AList [])]) + "{ f = (); }" `shouldParseInto` ADict (Map.fromList [("f", AList [])]) + "{ f = ( g ) }" `shouldParseInto` ADict (Map.fromList [("f", AList [AText "g"])]) + "{ f = (g) }" `shouldParseInto` ADict (Map.fromList [("f", AList [AText "g"])]) + + it "should parse dictionary with key, and value of dict" $ do + "{ h = { } }" `shouldParseInto` ADict (Map.fromList [("h", ADict Map.empty)]) + "{ h = { }; }" `shouldParseInto` ADict (Map.fromList [("h", ADict Map.empty)]) + "{ h = { another = dict } }" `shouldParseInto` ADict (Map.fromList [("h", ADict $ Map.fromList [("another", AText "dict")])]) + + it "should parse dictionary with multiple keys, and multiple value types" $ + "{ i = j; k = ( l ); m = { n = o } }" + `shouldParseInto` ADict + ( Map.fromList + [ ("i", AText "j") + , ("k", AList [AText "l"]) + , ("m", ADict $ Map.fromList [("n", AText "o")]) + ] + ) + + describe "parseAsciiValue" $ do + let shouldParseInto = parseMatch parseAsciiValue + it "should parse any ascii value type" $ do + -- Text + "a" `shouldParseInto` AText "a" + + -- List + "( )" `shouldParseInto` AList [] + "( a )" `shouldParseInto` AList [AText "a"] + "( a, )" `shouldParseInto` AList [AText "a"] + + -- Dictionary + "{ }" `shouldParseInto` ADict (Map.empty) + "{}" `shouldParseInto` ADict (Map.empty) + "{ b = c }" `shouldParseInto` ADict (Map.fromList [("b", AText "c")]) + "{ b = c; }" `shouldParseInto` ADict (Map.fromList [("b", AText "c")]) + "{ b = c; d = \"e\" }" `shouldParseInto` ADict (Map.fromList [("b", AText "c"), ("d", AText "e")]) + + describe "parsePbxProj" $ do + pbxprojFile <- runIO (TIO.readFile "test/Swift/Xcode/testdata/project.pbxproj") + it "should parse pbxproj.project" $ + case runParser parsePbxProj "" pbxprojFile of + Left _ -> expectationFailure "failed to parse" + Right result -> do + archiveVersion result `shouldBe` "1" + objectVersion result `shouldBe` "52" + rootObject result `shouldBe` "17874CD926C46B8500D16CA8" + classes result `shouldBe` Just (ADict Map.empty) + objects result `shouldNotBe` Nothing + + it "should parse pbxproj.project with just record fields" $ + case runParser parsePbxProj "" simplePbxProjFile of + Left _ -> expectationFailure "failed to parse" + Right result -> do + archiveVersion result `shouldBe` "1" + objectVersion result `shouldBe` "52" + rootObject result `shouldBe` "17874CD926C46B8500D16CA9" + classes result `shouldBe` Just (ADict Map.empty) + objects result + `shouldBe` Just + ( ADict $ + Map.fromList + [ + ( "172D94BF26C5D824008A4DB2" + , ADict $ + Map.fromList + [ ("isa", AText "PBXBuildFile") + , ("productRef", AText "172D94BE26C5D824008A4DB2") + ] + ) + ] + ) + + it "should fail when provided with non utf-8 encoding" $ + case runParser parsePbxProj "" unSupportedPbxProjFile of + Left errUnSupportFile -> errorBundlePretty errUnSupportFile `shouldContain` "expecting to have UTF8 Encoding!" + Right _ -> expectationFailure "should not parse this file!" diff --git a/test/Swift/Xcode/PbxprojSpec.hs b/test/Swift/Xcode/PbxprojSpec.hs new file mode 100644 index 000000000..2c6617705 --- /dev/null +++ b/test/Swift/Xcode/PbxprojSpec.hs @@ -0,0 +1,172 @@ +module Swift.Xcode.PbxprojSpec ( + spec, +) where + +import Data.Map.Strict qualified as Map +import Data.Maybe (fromMaybe) +import Data.Text (Text) +import Data.Text.IO qualified as TIO +import DepTypes (DepType (GitType, SwiftType), Dependency (..), VerConstraint (CEq)) +import GraphUtil (expectDeps, expectDirect, expectEdges) +import Strategy.Swift.PackageResolved (SwiftPackageResolvedFile (..), SwiftResolvedPackage (..)) +import Strategy.Swift.PackageSwift ( + SwiftPackageGitDepRequirement (..), + ) +import Strategy.Swift.Xcode.Pbxproj ( + XCRemoteSwiftPackageReference (..), + buildGraph, + swiftPackageReferencesOf, + ) +import Strategy.Swift.Xcode.PbxprojParser ( + AsciiValue (..), + PbxProj (..), + parsePbxProj, + ) +import Test.Hspec ( + Spec, + describe, + it, + runIO, + shouldBe, + ) +import Text.Megaparsec ( + parseMaybe, + ) + +mockUrl :: Text +mockUrl = "mock-url" + +mockId :: Text +mockId = "mock-id" + +emptyPbxProj :: PbxProj +emptyPbxProj = PbxProj "" "" "" Nothing Nothing + +makePbxProj :: AsciiValue -> PbxProj +makePbxProj obj = PbxProj "" "" "" Nothing (Just obj) + +makePbxProjWithXCRSwiftRef :: Text -> [(Text, AsciiValue)] -> PbxProj +makePbxProjWithXCRSwiftRef url req = makePbxProj (ADict $ Map.fromList [(mockId, xcrSwiftRef)]) + where + xcrSwiftRef = + ADict $ + Map.fromList + [ ("isa", AText "XCRemoteSwiftPackageReference") + , ("repositoryURL", AText url) + , ("requirement", ADict $ Map.fromList req) + ] + +makeXCRSwiftRef :: SwiftPackageGitDepRequirement -> XCRemoteSwiftPackageReference +makeXCRSwiftRef = XCRemoteSwiftPackageReference mockUrl + +makeGitDep :: Text -> Text -> Dependency +makeGitDep name c = Dependency GitType name (CEq <$> Just c) [] [] Map.empty + +makeSwiftDep :: Text -> Text -> Dependency +makeSwiftDep name c = Dependency SwiftType name (CEq <$> Just c) [] [] Map.empty + +spec :: Spec +spec = do + describe "swiftPackageReferencesOf" $ do + it "should be empty, when there are no XCRemoteSwiftPackageReference" $ do + let withoutIsa = makePbxProj (ADict $ Map.fromList [("A", AText "B")]) + swiftPackageReferencesOf withoutIsa `shouldBe` [] + + it "should be return XCRemoteSwiftPackageReference" $ do + -- Setup + let branchCase = + makePbxProjWithXCRSwiftRef + mockUrl + [ ("kind", AText "branch") + , ("branch", AText "develop") + ] + let revisionCase = + makePbxProjWithXCRSwiftRef + mockUrl + [ ("kind", AText "revision") + , ("revision", AText "05cd") + ] + let exactCase = + makePbxProjWithXCRSwiftRef + mockUrl + [ ("kind", AText "exactVersion") + , ("version", AText "1.2.3") + ] + let upToNextMajorCase = + makePbxProjWithXCRSwiftRef + mockUrl + [ ("kind", AText "upToNextMajorVersion") + , ("minimumVersion", AText "2.0.0") + ] + let upToNextMinorCase = + makePbxProjWithXCRSwiftRef + mockUrl + [ ("kind", AText "upToNextMinorVersion") + , ("minimumVersion", AText "3.0.0") + ] + let versionRangeCase = + makePbxProjWithXCRSwiftRef + mockUrl + [ ("kind", AText "versionRange") + , ("minimumVersion", AText "4.0.0") + , ("maximumVersion", AText "5.0.0") + ] + + -- Assert + swiftPackageReferencesOf branchCase `shouldBe` [makeXCRSwiftRef $ Branch "develop"] + swiftPackageReferencesOf revisionCase `shouldBe` [makeXCRSwiftRef $ Revision "05cd"] + swiftPackageReferencesOf exactCase `shouldBe` [makeXCRSwiftRef $ Exact "1.2.3"] + swiftPackageReferencesOf upToNextMajorCase `shouldBe` [makeXCRSwiftRef $ UpToNextMajor "2.0.0"] + swiftPackageReferencesOf upToNextMinorCase `shouldBe` [makeXCRSwiftRef $ UpToNextMinor "3.0.0"] + swiftPackageReferencesOf versionRangeCase `shouldBe` [makeXCRSwiftRef $ ClosedInterval ("4.0.0", "5.0.0")] + + describe "buildGraph" $ do + projFile <- runIO (TIO.readFile "test/Swift/Xcode/testdata/project.pbxproj") + let projContent = fromMaybe emptyPbxProj $ parseMaybe parsePbxProj projFile + + it "should build graph of direct dependencies, when package resolved is nothing" $ do + -- Setup + let graph = buildGraph projContent Nothing + let expectedDirectDeps = + [ makeGitDep "https://github.com/apple/example-package-deckofplayingcards" "main" + , makeGitDep "https://github.com/PopFlamingo/MyHTML.git" "2.0.0" + , makeGitDep "https://github.com/brightdigit/Spinetail.git" "97ad8ba7a43fac299ef88f3200fccf852c778b67" + , makeSwiftDep "https://github.com/vapor/vapor.git" ">=4.48.3 <=5.0.0" + , makeSwiftDep "https://github.com/MartinP7r/AckGen.git" "^0.1.0" + , makeSwiftDep "https://github.com/apple/swift-syntax" "~0.50400.0" + ] + + -- Assert + expectDirect expectedDirectDeps graph + expectDeps (expectedDirectDeps) graph + expectEdges [] graph + + it "should build graph of direct and deep dependencies, when package resolved exists" $ do + -- Setup + let expectedDirectDeps = [makeGitDep "dep-A" "some-rev-A"] + let expectedDeepDeps = [makeGitDep "dep-B" "some-rev-B"] + let mockProjContent = makePbxProjWithXCRSwiftRef "dep-A" [("kind", AText "exactVersion"), ("version", AText "1.2.5")] + let resolvedContent = + Just $ + SwiftPackageResolvedFile + 1 + [ SwiftResolvedPackage + "depA" + "dep-A" + Nothing + (Just "some-rev-A") + (Just "1.2.5") + , SwiftResolvedPackage + "depB" + "dep-B" + Nothing + (Just "some-rev-B") + (Just "1.2.6") + ] + -- Act + let graph = buildGraph mockProjContent resolvedContent + + -- Assert + expectDirect expectedDirectDeps graph + expectDeps (expectedDirectDeps ++ expectedDeepDeps) graph + expectEdges [] graph diff --git a/test/Swift/Xcode/testdata/project.pbxproj b/test/Swift/Xcode/testdata/project.pbxproj new file mode 100644 index 000000000..d700c355b --- /dev/null +++ b/test/Swift/Xcode/testdata/project.pbxproj @@ -0,0 +1,125 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 52; + objects = { + +/* Common Special Characters */ + 11111111111111111111111 = {isa = "※üAÆÁÂÄÀÅÃBCÇDEÉÊËÈЀFGHIÍÎÏÌJKLŁMNÑOŒÓÔÖÒØÕPQRSŠTÞUÚÛÜÙVWXYÝŸZŽaáâ ́äæà&å^~*@ãb\|{}[] ̆¦•cˇç ̧¢ˆ:,©¤d†‡° ̈÷$ ̇ıeéêëè8…—–=ð!¡ffi5flƒ4⁄gß`>«»‹›h ̋-iíîïìjkl<¬łm ̄−μ×n9ñ#oóôöœ ̨ò1½¼1aoøõp¶()%.·‰+±q?¿r® ̊sš§;76/£tþ3¾3 ̃™22uúûüùýÿ¥zž0"}; + +/* Begin PBXBuildFile section */ + 172D94BF26C5D824008A4DB2 /* Vapor in Frameworks */ = {isa = PBXBuildFile; productRef = 172D94BE26C5D824008A4DB2 /* Vapor */; }; + 17874CE526C46B8500D16CA8 /* ExampleProjectApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17874CE426C46B8500D16CA8 /* ExampleProjectApp.swift */; }; + 17874CE726C46B8500D16CA8 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17874CE626C46B8500D16CA8 /* ContentView.swift */; }; + 17874CE926C46B8800D16CA8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 17874CE826C46B8700D16CA8 /* Assets.xcassets */; }; + 17874CEC26C46B8800D16CA8 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 17874CEB26C46B8800D16CA8 /* Preview Assets.xcassets */; }; + 17874CF726C46B8800D16CA8 /* ExampleProjectTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17874CF626C46B8800D16CA8 /* ExampleProjectTests.swift */; }; + 17874D0226C46B8800D16CA8 /* ExampleProjectUITests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 17874D0126C46B8800D16CA8 /* ExampleProjectUITests.swift */; }; + 17874D1126C46CBA00D16CA8 /* AckGen in Frameworks */ = {isa = PBXBuildFile; productRef = 17874D1026C46CBA00D16CA8 /* AckGen */; }; + 17874D1326C46CBA00D16CA8 /* AckGenUI in Frameworks */ = {isa = PBXBuildFile; productRef = 17874D1226C46CBA00D16CA8 /* AckGenUI */; }; + 17874D1626C4877900D16CA8 /* SwiftSyntax in Frameworks */ = {isa = PBXBuildFile; productRef = 17874D1526C4877900D16CA8 /* SwiftSyntax */; }; + 17874D1826C4877900D16CA8 /* SwiftSyntaxBuilder in Frameworks */ = {isa = PBXBuildFile; productRef = 17874D1726C4877900D16CA8 /* SwiftSyntaxBuilder */; }; +/* End PBXBuildFile section */ + +/* ... some content excluded for brevity */ + +/* Begin XCRemoteSwiftPackageReference section */ + 170A463726ECEDEF002DDFB8 /* XCRemoteSwiftPackageReference "example-package-deckofplayingcards" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/apple/example-package-deckofplayingcards"; + requirement = { + branch = main; + kind = branch; + }; + }; + 170A463A26ED051A002DDFB8 /* XCRemoteSwiftPackageReference "MyHTML" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/PopFlamingo/MyHTML.git"; + requirement = { + kind = exactVersion; + version = 2.0.0; + }; + }; + 170A463D26ED054A002DDFB8 /* XCRemoteSwiftPackageReference "Spinetail" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/brightdigit/Spinetail.git"; + requirement = { + kind = revision; + revision = 97ad8ba7a43fac299ef88f3200fccf852c778b67; + }; + }; + 172D94BD26C5D824008A4DB2 /* XCRemoteSwiftPackageReference "vapor" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/vapor/vapor.git"; + requirement = { + kind = versionRange; + maximumVersion = 5.0.0; + minimumVersion = 4.48.3; + }; + }; + 17874D0F26C46CBA00D16CA8 /* XCRemoteSwiftPackageReference "AckGen" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/MartinP7r/AckGen.git"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 0.1.0; + }; + }; + 17874D1426C4877900D16CA8 /* XCRemoteSwiftPackageReference "swift-syntax" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/apple/swift-syntax"; + requirement = { + kind = upToNextMinorVersion; + minimumVersion = 0.50400.0; + }; + }; +/* End XCRemoteSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + 170A463826ECEDF0002DDFB8 /* DeckOfPlayingCards */ = { + isa = XCSwiftPackageProductDependency; + package = 170A463726ECEDEF002DDFB8 /* XCRemoteSwiftPackageReference "example-package-deckofplayingcards" */; + productName = DeckOfPlayingCards; + }; + 170A463B26ED051A002DDFB8 /* MyHTML */ = { + isa = XCSwiftPackageProductDependency; + package = 170A463A26ED051A002DDFB8 /* XCRemoteSwiftPackageReference "MyHTML" */; + productName = MyHTML; + }; + 170A463E26ED054A002DDFB8 /* Spinetail */ = { + isa = XCSwiftPackageProductDependency; + package = 170A463D26ED054A002DDFB8 /* XCRemoteSwiftPackageReference "Spinetail" */; + productName = Spinetail; + }; + 172D94BE26C5D824008A4DB2 /* Vapor */ = { + isa = XCSwiftPackageProductDependency; + package = 172D94BD26C5D824008A4DB2 /* XCRemoteSwiftPackageReference "vapor" */; + productName = Vapor; + }; + 17874D1026C46CBA00D16CA8 /* AckGen */ = { + isa = XCSwiftPackageProductDependency; + package = 17874D0F26C46CBA00D16CA8 /* XCRemoteSwiftPackageReference "AckGen" */; + productName = AckGen; + }; + 17874D1226C46CBA00D16CA8 /* AckGenUI */ = { + isa = XCSwiftPackageProductDependency; + package = 17874D0F26C46CBA00D16CA8 /* XCRemoteSwiftPackageReference "AckGen" */; + productName = AckGenUI; + }; + 17874D1526C4877900D16CA8 /* SwiftSyntax */ = { + isa = XCSwiftPackageProductDependency; + package = 17874D1426C4877900D16CA8 /* XCRemoteSwiftPackageReference "swift-syntax" */; + productName = SwiftSyntax; + }; + 17874D1726C4877900D16CA8 /* SwiftSyntaxBuilder */ = { + isa = XCSwiftPackageProductDependency; + package = 17874D1426C4877900D16CA8 /* XCRemoteSwiftPackageReference "swift-syntax" */; + productName = SwiftSyntaxBuilder; + }; +/* End XCSwiftPackageProductDependency section */ + + }; + rootObject = 17874CD926C46B8500D16CA8 /* Project object */; +} \ No newline at end of file diff --git a/test/Swift/testdata/Package.resolved b/test/Swift/testdata/Package.resolved new file mode 100644 index 000000000..e561ef68a --- /dev/null +++ b/test/Swift/testdata/Package.resolved @@ -0,0 +1,34 @@ +{ + "object": { + "pins": [ + { + "package": "grpc-swift", + "repositoryURL": "https://github.com/grpc/grpc-swift.git", + "state": { + "branch": null, + "revision": "9e464a75079928366aa7041769a271fac89271bf", + "version": "1.0.0" + } + }, + { + "package": "Opentracing", + "repositoryURL": "https://github.com/undefinedlabs/opentracing-objc", + "state": { + "branch": "master", + "revision": null, + "version": null + } + }, + { + "package": "Reachability", + "repositoryURL": "https://github.com/ashleymills/Reachability.swift", + "state": { + "branch": null, + "revision": null, + "version": "5.1.0" + } + } + ] + }, + "version": 1 +} From 08db396c9fda65257c9398a3e425d63aeba8470f Mon Sep 17 00:00:00 2001 From: Megh Date: Tue, 28 Sep 2021 13:37:44 -0600 Subject: [PATCH 9/9] updates changelog --- Changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Changelog.md b/Changelog.md index aedfb3a7f..48ce53a44 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,8 @@ # Spectrometer Changelog +## v2.16.0 +- Swift: Supports dependencies analysis for dependencies managed by Swift Package Manager. ([#354](https://github.com/fossas/spectrometer/pull/354)) + ## v2.15.24 - Leiningen: Executes `lein --version` before performing any analysis, to ensure Leiningen has performed its install tasks (done on its first invocation). ([#379](https://github.com/fossas/spectrometer/pull/379))