Merge pull request #26 from in-toto/debian

Add debian metadata and fixes for 0.1.0 release

.coveragerc

@@ -3,11 +3,13 @@
 branch = True
 parallel = True
 
-[report]
-exclude_lines =
-  pragma: no cover
-
 omit =
   */tests/*
+  */python?/*
   */python?.?/*
   */site-packages/*
+  */dist-packages/*
+
+[report]
+exclude_lines =
+  pragma: no cover

.gitignore

@@ -115,4 +115,7 @@ dmypy.json
 .pyre/
 
 # macOS
-.DS_Store
+.DS_Store
+
+# pycharm
+.idea

.travis.yml

@@ -1,4 +1,4 @@
-dist: xenial
+dist: bionic
 language: python
 
 matrix:
@@ -7,15 +7,8 @@ matrix:
         # tox environment, instead of using one Python version in Travis and
         # hoping that tox and pyenv run the tests in the desired versions.
         # https://github.com/travis-ci/travis-ci/issues/8363#issuecomment-355090242
-        - python: "2.7"
-          env: TOXENV=py27
-        - python: "3.5"
-          env: TOXENV=py35
-        - python: "3.6"
-          env: TOXENV=py36
-        - python: "3.7"
-          env: TOXENV=py37
-
+        - python: "3.9"
+          env: TOXENV=py39
 install:
     - pip install -U tox
     - pip install -U coveralls

README.md

@@ -23,7 +23,7 @@ git clone https://github.com/in-toto/apt-transport-in-toto.git
 # Install requirements
 pip install -r apt-transport-in-toto/requirements.txt
 # Install transport
-ln -s /usr/lib/apt/methods/intoto apt-transport-in-toto/intoto.py
+ln -s apt-transport-in-toto/intoto.py /usr/lib/apt/methods/intoto
 chmod 755 /usr/lib/apt/methods/intoto
 ```
 

data/intoto.conf

@@ -0,0 +1,12 @@
+APT::Intoto {
+  LogLevel {"20"};
+  Rebuilders {
+    "http://158.39.77.214";
+    "https://reproducible-builds.engineering.nyu.edu";
+  };
+  Layout {"/etc/intoto/root.layout"};
+  Keyids {
+    "88876A89E3D4698F83D3DB0E72E33CA3E0E04E46"
+  };
+  NoFail {"true"}
+};

debian/apt-transport-in-toto.install

@@ -0,0 +1,11 @@
+#!/usr/bin/dh-exec
+
+intoto.py => /usr/lib/apt/methods/intoto
+
+# TODO: Change root key
+data/intoto.conf => /etc/apt/apt.conf.d/intoto
+
+# TODO: Sign layout
+data/root.layout => /etc/intoto/root.layout
+
+# TODO: Copy signing and import into keychain, or use DD key?

debian/changelog

@@ -0,0 +1,5 @@
+apt-transport-in-toto (0.1.0) unstable; urgency=low
+
+  *  Initial Debian release. (Closes: #934143)
+
+ -- Lukas Puehringer <lukas.puehringer@nyu.edu>  Thu, 19 Dec 2019 13:37:12 -0100

debian/control

@@ -0,0 +1,45 @@
+Source: apt-transport-in-toto
+Section: utils
+Priority: optional
+Maintainer: in-toto developers <in-toto-dev@googlegroups.com>
+Uploaders:
+ Santiago Torres-Arias <santiago@nyu.edu>,
+ Lukas Puehringer <lukas.puehringer@nyu.edu>,
+ Holger Levsen <holger@debian.org>,
+ Vagrant Cascadian <vagrant@debian.org>,
+ Justin Cappos <jcappos@nyu.edu>,
+Build-Depends:
+ debhelper-compat (= 13),
+ dh-python,
+ dh-exec,
+ python3-all,
+ python3-requests,
+ python3-mock,
+ python3-coverage,
+ in-toto (>= 0.3.0),
+ gnupg2,
+Standards-Version: 4.5.1
+Rules-Requires-Root: no
+Homepage: https://in-toto.io
+Vcs-Git: https://github.com/in-toto/apt-transport-in-toto.git
+Vcs-Browser: https://github.com/in-toto/apt-transport-in-toto
+
+Package: apt-transport-in-toto
+Architecture: all
+Depends:
+  ${misc:Depends},
+  python3,
+  python3-requests,
+  python3-securesystemslib,
+  in-toto (>= 0.3.0),
+  gnupg2,
+Description: apt transport method for in-toto supply chain verification
+ apt-transport-in-toto provides a custom transport method for apt that fetches
+ and verifies signed build information from autonomous rebuilders upon package
+ installation.
+ .
+ It uses the supply chain security framework in-toto for its verification
+ protocol, to i.a. define trust relationships and exchange and verify build
+ information.
+ .
+ apt-transport-in-toto is developed at the Secure Systems Lab of NYU.

debian/copyright

@@ -0,0 +1,24 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: apt-transport-in-toto
+Source: https://github.com/in-toto/apt-transport-in-toto
+
+Files: *
+Copyright: 2018 New York University
+License: Apache-2.0
+ Copyright 2018 New York University
+ .
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+    http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the full text of the Apache-2.0
+ License can be found in the file
+ `/usr/share/common-licenses/Apache-2.0'.

debian/rules

@@ -0,0 +1,17 @@
+#!/usr/bin/make -f
+# You must remove unused comment lines for the released package.
+#export DH_VERBOSE = 1
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+#export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+
+%:
+	dh $@
+
+override_dh_auto_test:
+# Run upstream test suite unless skipped via DEB_BUILD_OPTIONS
+ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
+	python3-coverage run -m unittest discover
+	python3-coverage combine
+	python3-coverage report -m
+endif

debian/source/format

@@ -0,0 +1 @@
+3.0 (native)

http

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 """
 <Program Name>
   http

intoto.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 """
 <Program Name>
   intoto.py
@@ -110,15 +110,11 @@
 import requests
 import tempfile
 import shutil
+import queue as Queue # pylint: disable=import-error
+import subprocess
+import securesystemslib.gpg.functions
 
-if sys.version_info[0] == 2: # pragma: no cover
-  import Queue # pylint: disable=import-error
-  import subprocess32 as subprocess # pylint: disable=import-error
-else: # pragma: no cover
-  import queue as Queue # pylint: disable=import-error
-  import subprocess
-
-import in_toto.util
+import in_toto.exceptions
 import in_toto.verifylib
 import in_toto.models.link
 import in_toto.models.metadata
@@ -152,12 +148,18 @@
 # Upon reception we set INTERRUPTED to true, which may be used to gracefully
 # terminate.
 INTERRUPTED = False
+# TODO: Maybe we can replace the signal handler with a KeyboardInterrupt
+# try/except block in the main loop, for better readability.
 def signal_handler(*junk):
   # Set global INTERRUPTED flag telling worker threads to terminate
   logger.debug("Received SIGINT, setting global INTERRUPTED true")
   global INTERRUPTED
   INTERRUPTED = True
 
+# Global BROKENPIPE flag should be set to true, if a `write` or `flush` on a
+# stream raises a BrokenPipeError, to gracefully terminate reader threads.
+BROKENPIPE = False
+
 # APT Method Interface Message definition
 # The first line of each message is called the message header. The first 3
 # digits (called the Status Code) have the usual meaning found in the http
@@ -387,8 +389,11 @@ def read_one(stream):
 
   """
   message_str = ""
-  # Read from passed stream until apt sends us a SIGINT or EOF (see below)
-  while not INTERRUPTED: # pragma: no branch
+  # Read from stream until we get a SIGINT/BROKENPIPE, or reach EOF (see below)
+  # TODO: Do we need exception handling for the case where we select/read from
+  # a stream that was closed? If so, we should do it in the main loop for
+  # better readability.
+  while not (INTERRUPTED or BROKENPIPE): # pragma: no branch
     # Only read if there is data on the stream (non-blocking)
     if not select.select([stream], [], [], 0)[0]:
       continue
@@ -418,8 +423,21 @@ def write_one(message_str, stream):
   """Write the passed message to the passed stream.
 
   """
-  stream.write(message_str)
-  stream.flush()
+  try:
+    stream.write(message_str)
+    stream.flush()
+
+  except BrokenPipeError:
+    # TODO: Move exception handling to main loop for better readability
+    global BROKENPIPE
+    BROKENPIPE = True
+    logger.debug("BrokenPipeError while writing '{}' to '{}'.".format(
+        message_str, stream))
+    # Python flushes standard streams on exit; redirect remaining output
+    # to devnull to avoid another BrokenPipeError at shutdown
+    # See https://docs.python.org/3/library/signal.html#note-on-sigpipe
+    devnull = os.open(os.devnull, os.O_WRONLY)
+    os.dup2(devnull, sys.stdout.fileno())
 
 
 def notify_apt(code, message_text, uri):
@@ -491,7 +509,7 @@ def _intoto_parse_config(message_data):
     if field_name == "Config-Item" and field_value.startswith("APT::Intoto"):
       # Dissect config item
       logger.debug(field_value)
-      junk, junk, config_name, config_value = field_value.split("::")
+      _, _, config_name, config_value = field_value.split("::")
       # Strip leading "=", courtesy of apt config
       config_value = config_value.lstrip("=")
 
@@ -637,12 +655,11 @@ def _intoto_verify(message_data):
         global_info["config"]["Keyids"]))
     if gpg_home:
       logger.info("Use gpg keyring '{}' (apt config)".format(gpg_home))
-      layout_keys = in_toto.util.import_gpg_public_keys_from_keyring_as_dict(
-          keyids, gpg_home=gpg_home)
-    else: # pragma: no cover
+      layout_keys = securesystemslib.gpg.functions.export_pubkeys(
+        keyids, homedir=gpg_home)
+    else:  # pragma: no cover
       logger.info("Use default gpg keyring")
-      layout_keys = in_toto.util.import_gpg_public_keys_from_keyring_as_dict(
-          keyids)
+      layout_keys = securesystemslib.gpg.functions.export_pubkeys(keyids)
 
     logger.info("Run in-toto verification")
 
@@ -713,9 +730,9 @@ def loop():
   """
   # Start http transport in a subprocess
   # Messages from the parent process received on sys.stdin are relayed to the
-  # subprocesses stdin and vice versa, messages written to the subprocess's
+  # subprocess' stdin and vice versa, messages written to the subprocess'
   # stdout are relayed to the parent via sys.stdout.
-  http_proc = subprocess.Popen([APT_METHOD_HTTP], stdin=subprocess.PIPE,
+  http_proc = subprocess.Popen([APT_METHOD_HTTP], stdin=subprocess.PIPE, # nosec
       stdout=subprocess.PIPE, universal_newlines=True)
 
   # HTTP transport message reader thread to add messages from the http
@@ -730,8 +747,9 @@ def loop():
   apt_thread = threading.Thread(target=read_to_queue, args=(sys.stdin,
       apt_queue))
 
-  # Start reader threads. They will run until they see an EOF on their stream
-  # or the global INTERRUPTED flag is set to true (on SIGINT from apt).
+  # Start reader threads.
+  # They will run until they see an EOF on their stream, or the global
+  # INTERRUPTED or BROKENPIPE flags are set to true.
   http_thread.start()
   apt_thread.start()
 
@@ -766,16 +784,20 @@ def loop():
         logger.debug("Relay message")
         write_one(message, out)
 
-    # Exit when both threads have terminated (on EOF or INTERRUPTED)
+    # Exit when both threads have terminated (EOF, INTERRUPTED or BROKENPIPE)
     # NOTE: We do not check if there are still messages on the streams or
     # in the queue, assuming that there aren't or we can ignore them if both
     # threads have terminated.
     if (not apt_thread.is_alive() and not http_thread.is_alive()):
       logger.debug("The worker threads are dead. Long live the worker threads!"
           "Terminating.")
 
-      # If apt has sent us a SIGINT we relay it to the subprocess
-      if INTERRUPTED: # pragma: no branch
+      # If INTERRUPTED or BROKENPIPE are true it (likely?) means that apt
+      # sent a SIGINT or closed the pipe we were writing to. This means we
+      # should exit and tell the http child process to exit too.
+      # TODO: Could it be that the http child closed a pipe or sent a SITERM?
+      # TODO: Should we behave differently for the two signals?
+      if INTERRUPTED or BROKENPIPE: # pragma: no branch
         logger.debug("Relay SIGINT to http subprocess")
         http_proc.send_signal(signal.SIGINT)
 

requirements.txt

@@ -1,3 +1,3 @@
 in-toto
-subprocess32; python_version < '3'
 requests
+securesystemslib

tests/data/test.layout

@@ -2,13 +2,13 @@
  "signatures": [
   {
    "keyid": "88876a89e3d4698f83d3db0e72e33ca3e0e04e46",
-   "other_headers": "04000108001d16210488876a89e3d4698f83d3db0e72e33ca3e0e04e4605025c348c50",
-   "signature": "2a5e5f62641c19e998ef0d3d41edbce64bc6c70ec8a10c271ca282340ce5ea5f56644911e55e1234837e6a468fe54a5fac224d1bae902bb46da9552a464b95304062fa18b873fee3f536d490dc762dc46b27cfb0058378b597136350da46d1dac8488137a1a048a0c1300c72980a627267ef49570e546c7b967786f663c4ebc6ed47545e34a7d2f89013e7c4af02ef79e7a2a345cf4aa8d761b1762a45f4fda266449cad36eeee22d24c426fba3d38d5377b2d2a7d62b188ae52ebd8eb71e2ec69eab3062c71f513c2f7999f8360a3e9784fc6b8fbd9cbc367020ef6f4394b8ba8e2b49fdbb8dfc4a241d8ae53c2ba3ff1f2e638b254a0110e0bc5e52c8b6785"
+   "other_headers": "04000108001d16210488876a89e3d4698f83d3db0e72e33ca3e0e04e460502600eaa72",
+   "signature": "4e03ee1b0ad69bc581b6e91bd7760db932020b1a3233bb09b7cf4bb118187f1c2ab19c43169399ef609835f829164f86da440dce4299d5dac027f5fddf1ef6ae6b6828de1d2fac34415a318c3b235fdc352543837a70d5bc2693af25a48c76c0d37c31ce9fdd2aa6f6c0f3cdb03db6ab710ed4e3f10ca51ca8a39fd1d6237f0909e0cea257b579b66db13b3451a597011670bf153b406c5b7218762e31ea6ec9294cc4058da5639970c069281cb8bfa3adaff505c3967dc55128be85150dc1d041f0e0bacd2dc38e535534228cab86f0954205ca24453a99a0f1758ec386100c9a4d370849728e42fc2720a6fa8bda3984eecb5eba28ee8008e9e7ae33a9a4aa"
   }
  ],
  "signed": {
   "_type": "layout",
-  "expires": "2021-01-06T18:30:57Z",
+  "expires": "2030-01-01T00:00:00Z",
   "inspect": [
    {
     "_type": "inspection",

tests/data/test.layout.docker

@@ -2,13 +2,13 @@
  "signatures": [
   {
    "keyid": "88876a89e3d4698f83d3db0e72e33ca3e0e04e46",
-   "other_headers": "04000108001d16210488876a89e3d4698f83d3db0e72e33ca3e0e04e4605025c348c1c",
-   "signature": "3b7ed41185c6e7c7ee8845a2b0604201347e383243ff21687580231c9b2e537b92897f4a579a7e63fe654b0cb21fd971b5dd21dfea60c75cacb2e6acbcb4642bf1515e032953215a3d0371d16ea4e10753ab1a74f57ebe2bb7f60185fc8ff9aadb1a5f3de63b3823ef158a199a58cd5adbbf7d240796a9966e365c3bcbadac9d967b3e072e77ed065a6d601ceeed06a398b665f692f5151575367331cb67b89081a4862cf2d8b0d76722935f4c45566214bf876e58c938e39b704c6d00bd228587eed8d22cc18665d4923219192e312e2259a607e09a265775ba83edc9dbff85e04864629f1bd12999ad2a2916c2e2c0c485a5350461105f29a085ee41f2da53"
+   "other_headers": "04000108001d16210488876a89e3d4698f83d3db0e72e33ca3e0e04e460502600eaa7a",
+   "signature": "0d09b045bb7baabf8a099830e24a0e36454253f811fda0d0b12fc99c1bcace9bd79aee921e1ebecbc7134fc3c0de584b38aca2e793ce2a77c1c66d91db33817ba8ca1021a77a9d0cce535761ce1c247e20e46637737cd2a954de4d572b2a3100756f150cb7544772e4a456b392219a694b1a41ce56d7016056a1b00da585db7303db706b7942d08e33b13b55569c541d98f8b6d19dfb8847d71cf853196f69ff30571ba875c8a673943262209c2ea725379590ff95ab179529874860da0aba6eba5c4299af7e5efaa13969472a08c6c3d0cff26b67b791ac1aca186d5df979588268e2593cc100c24cc0d6ca286eb321b64fea105733bb45cbfa6046b89640f0"
   }
  ],
  "signed": {
   "_type": "layout",
-  "expires": "2021-01-06T18:30:57Z",
+  "expires": "2030-01-01T00:00:00Z",
   "inspect": [
    {
     "_type": "inspection",

tests/measure_coverage.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 """
 <Program Name>
   measure_coverage.py

tests/serve_metadata.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python3
 """
 <Program Name>
   serve_metadata.py