Velocidex · mgreen27 · Sep 27, 2023 · Sep 27, 2023
diff --git a/content/artifact_references/pages/generic.forensic.sqlitehunter.md b/content/artifact_references/pages/generic.forensic.sqlitehunter.md
diff --git a/content/artifact_references/pages/generic.system.hostsfile.md b/content/artifact_references/pages/generic.system.hostsfile.md
@@ -0,0 +1,134 @@
+---
+title: Generic.System.HostsFile
+hidden: true
+tags: [Client Artifact]
+---
+
+The system hosts file maps hostnames to IP addresses. In some cases,
+entries in this file take precedence and overrides the results from
+the system DNS service.
+
+The file is a simple text file, with one line per IP address. Each
+whitespace-separated word following the IP address is a hostname.
+The Linux man page refers to the the first hostname as *canonical_hostname*,
+and any following words as *aliases*. They are treated the same by this
+artifact.
+
+The hosts file is typically present on all Linux-based systems (including macOS),
+with entries for localhost. The same file format is also supported on Windows.
+
+The source *Hosts* returns each line in each hosts file that matches
+the glob parameters for address and hostname. The hostname and aliases
+are combined in a single column *Hostnames*. Columns returned:
+
+- OSPath
+- Hostnames
+- Comment
+
+Only comments that follows the hostname on the same line are captured in Comment.
+Comments on their own lines are ignored.
+
+A second source *HostsFlattened* provides a flattened result, with each row
+containing an IP address and a single hostname.
+
+This artifact also exports a function `parse_hostsfile()` that returns Hostname
+and Aliases individually.
+
+
+<pre><code class="language-yaml">
+name: Generic.System.HostsFile
+description: |
+  The system hosts file maps hostnames to IP addresses. In some cases,
+  entries in this file take precedence and overrides the results from
+  the system DNS service.
+
+  The file is a simple text file, with one line per IP address. Each
+  whitespace-separated word following the IP address is a hostname.
+  The Linux man page refers to the the first hostname as *canonical_hostname*,
+  and any following words as *aliases*. They are treated the same by this
+  artifact.
+
+  The hosts file is typically present on all Linux-based systems (including macOS),
+  with entries for localhost. The same file format is also supported on Windows.
+
+  The source *Hosts* returns each line in each hosts file that matches
+  the glob parameters for address and hostname. The hostname and aliases
+  are combined in a single column *Hostnames*. Columns returned:
+
+  - OSPath
+  - Hostnames
+  - Comment
+
+  Only comments that follows the hostname on the same line are captured in Comment.
+  Comments on their own lines are ignored.
+
+  A second source *HostsFlattened* provides a flattened result, with each row
+  containing an IP address and a single hostname.
+
+  This artifact also exports a function `parse_hostsfile()` that returns Hostname
+  and Aliases individually.
+
+reference:
+  - https://manpages.debian.org/bookworm/manpages/hosts.5.en.html
+
+export: |
+  LET _parse_hostsfile(OSPath) = SELECT parse_string_with_regex(
+     string=Line,
+     regex='''^[\t ]*(?P&lt;Address&gt;[^\s#]+)[\t ]+(?P&lt;Hostname&gt;[^\s#]+)(?P&lt;Aliases&gt;[^#\n\r]+)?(?:[\t ]*#(?P&lt;Comment&gt;.+))?''') AS Parsed
+  FROM parse_lines(filename=OSPath)
+  WHERE Parsed.Address
+
+  LET parse_hostsfile(OSPath) = SELECT Parsed.Address AS Address,
+     Parsed.Hostname AS Hostname,
+     filter(list=split(sep='''\s+''', string=Parsed.Aliases), regex='.') AS Aliases,
+
+     /* Remove any whitespace between comment character and comment: */
+     regex_replace(re='''^\s+''', source=Parsed.Comment, replace='$1') AS Comment
+  FROM _parse_hostsfile(OSPath=OSPath)
+
+  LET Files = SELECT OSPath FROM glob(globs=hostsFileGlobs.HostsFileGlobs)
+
+  LET HostsFiles = SELECT * FROM foreach(row=Files, query={
+    SELECT OSPath, Address, Hostname, Aliases, Comment
+    FROM parse_hostsfile(OSPath=OSPath)
+  })
+
+parameters:
+  - name: hostsFileGlobs
+    description: Globs to find hosts files
+    type: csv
+    default: |
+        HostsFileGlobs
+        C:\Windows\System32\drivers\etc\hosts
+        /etc/hosts
+  - name: HostnameRegex
+    description: Hostname or aliases to match
+    default: .
+    type: regex
+  - name: AddressRegex
+    description: IP addresses to match
+    default: .
+    type: regex
+
+sources:
+  - name: Hosts
+    query: |
+      SELECT OSPath, Address,
+        (Hostname, ) + Aliases AS Hostname,
+        Comment
+      FROM HostsFiles
+      WHERE Hostname =~ HostnameRegex
+        AND Address =~ AddressRegex
+
+  - name: HostsFlattened
+    query: |
+      SELECT OSPath, Address, Hostname, Comment
+      FROM flatten(query={
+        SELECT OSPath, Address, (Hostname, ) + Aliases AS Hostname, Comment
+        FROM HostsFiles
+      })
+      WHERE Address =~ AddressRegex
+        AND Hostname =~ HostnameRegex
+
+</code></pre>
+
diff --git a/content/artifact_references/pages/linux.debian.aptsources.md b/content/artifact_references/pages/linux.debian.aptsources.md
@@ -22,7 +22,7 @@ deb indicates a source for binary packages, and deb-src instructs APT where
 to find source code for packages.
 
 `*.sources` files (deb822-style format) are in the form of key–value
-lines, and as opposed to the one–line format, they can contain
+lines, and as opposed to the one–line format, they may contain
 multiple URIs, components and types (deb/deb-src), along with
 embedded GPG keys. Example:
 
@@ -33,8 +33,9 @@ Suites: unstable
 Components: main contrib non-free
 ```
 
-The exported function parse_aptsources(OSPath, flatten) parses
-both formats and returns a (optionally flattened) table with
+The exported function `parse_aptsources(OSPath, flatten)` parses
+both formats and returns an (optionally flattened) table with
+
  - OSPath
  - Types (deb/deb-src)
  - Components (e.g. main/contrib/non-free/restricted,universe)
@@ -43,30 +44,36 @@ both formats and returns a (optionally flattened) table with
  - _Transport (e.g. http/https/file/cdrom/ftp)
  - URIs (e.g. http://us.archive.ubuntu.com/ubuntu/)
 
-Any option is added to an individual column. Typical options are
+Any option is added to an individual column. The most common options
+are
+
  - Architectures (e.g. amd64/i386/armel)
  - Signed-By (e.g. /usr/share/keyrings/osquery.gpg)
 
 All known option names are transformed to the plural PascalCase
 variants as listed in the sources.list man page. Any undocumented
 options will still be included in the results, with names unchanged.
 Options in the one-line format of the form "lang+=de"/"arch-=i386"
-will be in columns like "Languages-Add"/"Architectures-Remove", matching
-the option names having the same effect in deb822.
+will be put in columns like "Languages-Add"/"Architectures-Remove",
+matching the option names having the same effect in deb822.
 
 Entries in deb822 sources files may be disabled by including
 "Enabled: no" instead of commenting out all lines. If this field
 is not present with a falsly value, the entry is enabled. Use the
 exported functions DebTrue()/DebFalse() to correctly parse all
-accepted true/false strings, or use the VQL suggestion "Enabled"
-to filter on this column (true), if present.
+accepted true/false strings, or use the VQL suggestion "Only enabled
+sources" to filter on this column (true), if present.
 
 If the GPG key is embedded in a .sources file, the whole GPG key
 will be included in the cell. Otherwise the value will be a file
-path.
+path. Use the VQL suggestion "Hide embedded GPG keys" to replace
+embedded GPG keys with "(embedded)" in the results. In order to
+inspect the keys themselves (files or embedded data), use the
+exchange artifact Linux.Debian.GPGKeys.
 
-If flatten is False, multi–value fields (like Components) will
-be combined in a single-space-separated string in each row.
+If the function parameter "flatten" is False, multi–value fields
+(like Components) will be combined in a single space-separated
+string in each row.
 
 In addition to the two apt sources tables, a third table correlates
 information from InRelease and Release files to provide additional
@@ -95,7 +102,7 @@ description: |
   to find source code for packages.
 
   `*.sources` files (deb822-style format) are in the form of key–value
-  lines, and as opposed to the one–line format, they can contain
+  lines, and as opposed to the one–line format, they may contain
   multiple URIs, components and types (deb/deb-src), along with
   embedded GPG keys. Example:
 
@@ -106,8 +113,9 @@ description: |
   Components: main contrib non-free
   ```
 
-  The exported function parse_aptsources(OSPath, flatten) parses
-  both formats and returns a (optionally flattened) table with
+  The exported function `parse_aptsources(OSPath, flatten)` parses
+  both formats and returns an (optionally flattened) table with
+
    - OSPath
    - Types (deb/deb-src)
    - Components (e.g. main/contrib/non-free/restricted,universe)
@@ -116,30 +124,36 @@ description: |
    - _Transport (e.g. http/https/file/cdrom/ftp)
    - URIs (e.g. http://us.archive.ubuntu.com/ubuntu/)
 
-  Any option is added to an individual column. Typical options are
+  Any option is added to an individual column. The most common options
+  are
+
    - Architectures (e.g. amd64/i386/armel)
    - Signed-By (e.g. /usr/share/keyrings/osquery.gpg)
 
   All known option names are transformed to the plural PascalCase
   variants as listed in the sources.list man page. Any undocumented
   options will still be included in the results, with names unchanged.
   Options in the one-line format of the form "lang+=de"/"arch-=i386"
-  will be in columns like "Languages-Add"/"Architectures-Remove", matching
-  the option names having the same effect in deb822.
+  will be put in columns like "Languages-Add"/"Architectures-Remove",
+  matching the option names having the same effect in deb822.
 
   Entries in deb822 sources files may be disabled by including
   "Enabled: no" instead of commenting out all lines. If this field
   is not present with a falsly value, the entry is enabled. Use the
   exported functions DebTrue()/DebFalse() to correctly parse all
-  accepted true/false strings, or use the VQL suggestion "Enabled"
-  to filter on this column (true), if present.
+  accepted true/false strings, or use the VQL suggestion "Only enabled
+  sources" to filter on this column (true), if present.
 
   If the GPG key is embedded in a .sources file, the whole GPG key
   will be included in the cell. Otherwise the value will be a file
-  path.
+  path. Use the VQL suggestion "Hide embedded GPG keys" to replace
+  embedded GPG keys with "(embedded)" in the results. In order to
+  inspect the keys themselves (files or embedded data), use the
+  exchange artifact Linux.Debian.GPGKeys.
 
-  If flatten is False, multi–value fields (like Components) will
-  be combined in a single-space-separated string in each row.
+  If the function parameter "flatten" is False, multi–value fields
+  (like Components) will be combined in a single space-separated
+  string in each row.
 
   In addition to the two apt sources tables, a third table correlates
   information from InRelease and Release files to provide additional
@@ -380,10 +394,6 @@ export: |
             columns='Section',
             regex='^ #', record_regex='''\n{2,}'''
         )
-        /* Sections may be empty due to several newlines or comments on their own
-           separated by newlines. Ensure that at least one field is present
-           (URIs are mandatory): */
-        WHERE URIs
 
         LET Deb822_Flattened_(OSPath) = SELECT * FROM foreach(
             row=Deb822Sections(OSPath=OSPath),
@@ -394,6 +404,9 @@ export: |
                 )
             })}
         )
+        /* DEB822_Sections() may produce empty rows. Exclude these by filtering
+           for a required column, like URIs: */
+        WHERE URIs
 
         /* Parse a deb822 sources file with options in individual columns.
            Note that, as opposed to DebOneLine and Deb822_Flattened, this
@@ -406,6 +419,7 @@ export: |
                 column='Contents'
             )}
         )
+        WHERE URIs
 
         /* Parse a deb822 sources file with options in individual columns, flattened: */
         LET Deb822_Flattened(OSPath) = SELECT * FROM flatten(query={
@@ -461,14 +475,34 @@ sources:
       - type: vql_suggestion
         name: Only enabled sources
         template: |
-            SELECT * FROM source(artifact='Custom.Linux.Debian.AptSources/Sources')
-            WHERE get(field='Enabled', default='yes') =~ '(?i)^(?:yes|true|with|on|enable)$'
+            /*
+            # Sources (enabled only)
+            */
+            SELECT * FROM source()
+            WHERE Enabled =~ '(?i)^(?:yes|true|with|on|enable)$' || true
 
       - type: vql_suggestion
         name: Trusted sources (apt-secure bypassed)
         template: |
-            SELECT * FROM source(artifact='Custom.Linux.Debian.AptSources/Sources')
-            WHERE get(field='Trusted', default='') =~ '(?i)^(?:yes|true|with|on|enable)$'
+            /*
+            # "Trusted" sources (apt-secure bypassed)
+
+            When the Trusted option is true, apt does not verify the GPG
+            signature of the Release files of the repository, and it also
+            doe not warn about this.
+            */
+            SELECT * FROM source()
+            WHERE Trusted =~ '(?i)^(?:yes|true|with|on|enable)$' || false
+
+      - type: vql_suggestion
+        name: Hide embedded GPG keys
+        template: |
+            /*
+            # Sources (embedded GPG keys hidden)
+            */
+            SELECT *, if(condition=get(field='Signed-By')=~'BEGIN PGP PUBLIC KEY',
+                then='(embedded)', else=get(field='Signed-By')) AS `Signed-By`
+                FROM source()
 
   - name: SourcesFlattened
     query: |

diff --git a/content/artifact_references/pages/server.internal.tooldependencies.md b/content/artifact_references/pages/server.internal.tooldependencies.md
@@ -20,19 +20,19 @@ description: |
 
 tools:
   - name: VelociraptorWindows
-    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-windows-amd64.exe
+    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-2-windows-amd64.exe
     serve_locally: true
-    version: 0.7.0
+    version: 0.7.0-2
 
   - name: VelociraptorWindows_x86
-    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-windows-386.exe
+    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-2-windows-386.exe
     serve_locally: true
-    version: 0.7.0
+    version: 0.7.0-2
 
   - name: VelociraptorLinux
-    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-linux-amd64-musl
+    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-2-linux-amd64-musl
     serve_locally: true
-    version: 0.7.0
+    version: 0.7.0-2
 
   # On MacOS we can not embed the config in the binary so we use a
   # shell script stub instead. See
@@ -44,14 +44,14 @@ tools:
     serve_locally: true
 
   - name: VelociraptorWindowsMSI
-    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-windows-amd64.msi
+    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-2-windows-amd64.msi
     serve_locally: true
-    version: 0.7.0
+    version: 0.7.0-2
 
   - name: VelociraptorWindows_x86MSI
-    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-windows-386.msi
+    url: https://github.com/Velocidex/velociraptor/releases/download/v0.7.0/velociraptor-v0.7.0-2-windows-386.msi
     serve_locally: true
-    version: 0.7.0
+    version: 0.7.0-2
 
 </code></pre>