Merge remote-tracking branch 'upstream/master' into hash

* upstream/master: (23 commits)
  Check for HTTP status 200 in fetchAndParse.
  Run everything through gofmt.
  Tweak the imports to make goimports happier.
  Initialize client if passed nil (google#1265)
  Have the caller supply the *http.Client used in logclient.go.
  Review fixes
  Use ASN1 lib to transform TLS encoded list into a Octet String
  Move SCTListLen Check to beginning
  Rework patch
  Add support for serializing SCT Lists
  Get rid of the Java MerkleLeafType enum.
  Add a missing entry to the .gitignore file.
  Get rid of the TLSSerializer class. Make the low-level serialization functions stand-alone ones in a namespace rather than members of a class.
  6962-bis: Implement OID parsing/serialization.
  Do not use std::move in "return" statements.
  Improve stats comments and add additional inc.
  Add additonal stats to outer FixAndLog wrapper.
  [golang] Merkle verifier (google#1161)
  Ignore clang warnings due to unused stuff caused by glog (google#1253)
  Remove a few unnecessary uses of std::move.
  ...

.gitignore

@@ -68,6 +68,7 @@
 /cpp/monitoring/registry_test
 /cpp/net/url_fetcher_test
 /cpp/proto/serializer_test
+/cpp/proto/serializer_v2_test
 /cpp/server/blob-server
 /cpp/server/ct-dns-server
 /cpp/server/ct-mirror

.travis.yml

@@ -13,7 +13,6 @@ addons:
   apt:
     sources: &common_sources
       - ubuntu-toolchain-r-test
-      - llvm-toolchain-precise-3.6
       - kalakris-cmake
     packages: &common_packages
       - autoconf
@@ -31,31 +30,28 @@ matrix:
       env: SSL=openssl
     - compiler: clang
       os: linux
-      env: SSL=boringssl REAL_CC=clang-3.6 REAL_CXX=clang++-3.6
+      env: SSL=boringssl LLVM=3.6.2
       addons:
         apt:
           sources: *common_sources
           packages:
           - *common_packages
-          - clang-3.6
     - compiler: clang
       os: linux
-      env: SSL=openssl REAL_CC=clang-3.6 REAL_CXX=clang++-3.6
+      env: SSL=openssl LLVM=3.6.2
       addons:
         apt:
           sources: *common_sources
           packages:
           - *common_packages
-          - clang-3.6
     - compiler: clang
       os: linux
-      env: SSL=openssl REAL_CC=clang-3.6 REAL_CXX=clang++-3.6 SANITIZE="-fsanitize=address -fno-omit-frame-pointer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=1" LSAN_OPTIONS="suppressions=${HOME}/lsan.supp"
+      env: SSL=openssl LLVM=3.6.2 SANITIZE="-fsanitize=address -fno-omit-frame-pointer" ASAN_OPTIONS="detect_leaks=1:check_initialization_order=1" LSAN_OPTIONS="suppressions=${HOME}/lsan.supp"
       addons:
         apt:
           sources: *common_sources
           packages:
           - *common_packages
-          - clang-3.6
   exclude:
     - compiler: gcc
       os: linux
@@ -95,6 +91,13 @@ before_install:
  - echo "protoc.bin = ${PROTOC}" >> local.properties
 
 install:
+# Workaround LLVM's apt mirror being feeble and underpowered
+ - |
+   if [ "$LLVM" != "" ]; then
+     CLANG_URL="http://llvm.org/releases/${LLVM}/clang+llvm-${LLVM}-x86_64-linux-gnu-ubuntu-14.04.tar.xz"
+     mkdir /tmp/clang && travis_retry wget --quiet -O - ${CLANG_URL} | tar --strip-components=1 -xJ -C /tmp/clang
+     export PATH="/tmp/clang/bin:${PATH}"
+    fi
  - autoconf --version
  - pushd ..
  - mkdir -p ct

Makefile.am

@@ -105,6 +105,7 @@ TESTS = \
 	cpp/monitoring/gauge_test \
 	cpp/monitoring/registry_test \
 	cpp/proto/serializer_test \
+	cpp/proto/serializer_v2_test \
 	cpp/server/proxy_test \
 	cpp/util/bignum_test \
 	cpp/util/etcd_delete_test \
@@ -189,6 +190,8 @@ cpp_libcore_a_SOURCES = \
 	cpp/net/url_fetcher.cc \
 	cpp/proto/cert_serializer.cc \
 	cpp/proto/serializer.cc \
+	cpp/proto/serializer_v2.cc \
+	cpp/proto/tls_encoding.cc \
 	cpp/server/metrics.cc \
 	cpp/server/proxy.cc \
 	cpp/server/server.cc \
@@ -708,6 +711,17 @@ cpp_proto_serializer_test_SOURCES = \
 	cpp/proto/serializer_test.cc \
 	cpp/util/util.cc
 
+cpp_proto_serializer_v2_test_LDADD = \
+	cpp/libcore.a \
+	cpp/libtest.a \
+	$(evhtp_LIBS) \
+	$(libevent_LIBS) \
+	-lprotobuf -lcrypto
+cpp_proto_serializer_v2_test_SOURCES = \
+	cpp/proto/serializer_v2.cc \
+	cpp/proto/serializer_v2_test.cc \
+	cpp/util/util.cc
+
 cpp_server_proxy_test_LDADD = \
 	cpp/libcore.a \
 	cpp/libtest.a \

cpp/client/async_log_client.cc

@@ -17,6 +17,7 @@ using cert_trans::CertChain;
 using cert_trans::PreCertChain;
 using cert_trans::URL;
 using cert_trans::UrlFetcher;
+using cert_trans::serialization::DeserializeResult;
 using ct::DigitallySigned;
 using ct::MerkleAuditProof;
 using ct::SignedCertificateTimestamp;

cpp/client/ct.cc

@@ -127,6 +127,8 @@ using cert_trans::ScopedRSA;
 using cert_trans::ScopedX509;
 using cert_trans::ScopedX509_NAME;
 using cert_trans::TbsCertificate;
+using cert_trans::serialization::SerializeResult;
+using cert_trans::serialization::DeserializeResult;
 using ct::LogEntry;
 using ct::MerkleAuditProof;
 using ct::SSLClientCTData;

cpp/client/ssl_client.cc

@@ -13,6 +13,7 @@
 #include "merkletree/serial_hasher.h"
 #include "proto/serializer.h"
 
+using cert_trans::serialization::DeserializeResult;
 using ct::LogEntry;
 using ct::SSLClientCTData;
 using ct::SignedCertificateTimestamp;

cpp/log/cert_checker_test.cc

@@ -628,9 +628,9 @@ TEST_F(CertCheckerTest, TestDsaPrecertChain) {
 
   EXPECT_OK(checker_.CheckPreCertChain(&pre_chain, &issuer_key_hash, &tbs));
   // Added a root CA.
-  EXPECT_EQ(2, pre_chain.Length());
+  EXPECT_EQ(2U, pre_chain.Length());
   // And set a SHA256 HASH
-  EXPECT_EQ(32, issuer_key_hash.size());
+  EXPECT_EQ(32U, issuer_key_hash.size());
   // And the TBS fields
   EXPECT_FALSE(tbs.empty());
 }

cpp/log/cert_test.cc

@@ -240,7 +240,7 @@ unique_ptr<Cert> ReadCertFromFile(const string& filename) {
       << ". Wrong --test_srcdir?";
   unique_ptr<Cert> cert(Cert::FromPemString(content));
   CHECK(cert.get());
-  return std::move(cert);
+  return cert;
 }
 
 class CertTest : public ::testing::Test {
@@ -504,7 +504,7 @@ TEST_F(CertTest, IllegalSignatureAlgorithmParameter) {
 TEST_F(CertTest, TestSubjectAltNames) {
   vector<string> sans;
   EXPECT_OK(google_cert_->SubjectAltNames(&sans));
-  EXPECT_EQ(44, sans.size());
+  EXPECT_EQ(44U, sans.size());
   EXPECT_EQ("*.google.com", sans[0]);
   EXPECT_EQ("*.android.com", sans[1]);
   EXPECT_EQ("youtubeeducation.com", sans[43]);
@@ -513,7 +513,7 @@ TEST_F(CertTest, TestSubjectAltNames) {
 TEST_F(CertTest, SPKI) {
   const StatusOr<string> spki(leaf_cert_->SPKI());
   EXPECT_OK(spki.status());
-  EXPECT_EQ(162, spki.ValueOrDie().size());
+  EXPECT_EQ(162U, spki.ValueOrDie().size());
   EXPECT_EQ("Ojz4hdfbFTowDio/KDGC4/pN9dy/EBfIAsnO2yDbKiE=",
             util::ToBase64(Sha256Hasher::Sha256Digest(spki.ValueOrDie())));
 }

cpp/log/cms_verifier.cc

@@ -4,7 +4,6 @@
 #include "util/cms_scoped_types.h"
 #include "util/openssl_scoped_types.h"
 
-using std::move;
 using std::string;
 using std::unique_ptr;
 using util::Status;
@@ -203,7 +202,7 @@ unique_ptr<Cert> CmsVerifier::UnpackCmsSignedCertificate(
     LOG_OPENSSL_ERRORS(ERROR);
   }
 
-  return move(cert);
+  return cert;
 }
 
 unique_ptr<Cert> CmsVerifier::UnpackCmsSignedCertificate(
@@ -228,7 +227,7 @@ unique_ptr<Cert> CmsVerifier::UnpackCmsSignedCertificate(
     LOG_OPENSSL_ERRORS(ERROR);
   }
 
-  return move(cert);
+  return cert;
 }
 
 }  // namespace cert_trans

cpp/log/cms_verifier_test.cc

@@ -56,7 +56,7 @@ unique_ptr<Cert> ReadCertFromFile(const string& filename) {
       << ". Wrong --test_srcdir?";
   unique_ptr<Cert> cert(Cert::FromPemString(content));
   CHECK(cert.get());
-  return std::move(cert);
+  return cert;
 }
 
 class CmsVerifierTest : public ::testing::Test {

cpp/log/file_db.cc

@@ -14,6 +14,7 @@
 #include "proto/serializer.h"
 #include "util/util.h"
 
+using cert_trans::serialization::DeserializeResult;
 using std::chrono::milliseconds;
 using std::lock_guard;
 using std::make_pair;

cpp/log/leveldb_db.cc

@@ -12,6 +12,7 @@
 #include "proto/serializer.h"
 #include "util/util.h"
 
+using cert_trans::serialization::DeserializeResult;
 using std::chrono::milliseconds;
 using std::lock_guard;
 using std::make_pair;

cpp/log/log_signer.cc

@@ -12,6 +12,8 @@
 #include "util/util.h"
 
 using cert_trans::Verifier;
+using cert_trans::serialization::SerializeResult;
+using cert_trans::serialization::DeserializeResult;
 using ct::DigitallySigned;
 using ct::LogEntry;
 using ct::LogEntryType;

cpp/log/log_signer.h

@@ -53,7 +53,8 @@ class LogSigner : public cert_trans::Signer {
   SignResult SignTreeHead(ct::SignedTreeHead* sth) const;
 
  private:
-  static SignResult GetSerializeError(SerializeResult result);
+  static SignResult GetSerializeError(
+      cert_trans::serialization::SerializeResult result);
 };
 
 class LogSigVerifier : public cert_trans::Verifier {
@@ -103,9 +104,11 @@ class LogSigVerifier : public cert_trans::Verifier {
   VerifyResult VerifySTHSignature(const ct::SignedTreeHead& sth) const;
 
  private:
-  static VerifyResult GetSerializeError(SerializeResult result);
+  static VerifyResult GetSerializeError(
+      cert_trans::serialization::SerializeResult result);
 
-  static VerifyResult GetDeserializeSignatureError(DeserializeResult result);
+  static VerifyResult GetDeserializeSignatureError(
+      cert_trans::serialization::DeserializeResult result);
 };
 
 #endif  // CERT_TRANS_LOG_LOG_SIGNER_H_

cpp/log/log_signer_test.cc

@@ -14,6 +14,8 @@
 
 namespace {
 
+using cert_trans::serialization::SerializeResult;
+using cert_trans::serialization::DeserializeResult;
 using ct::LogEntry;
 using ct::SignedCertificateTimestamp;
 using ct::DigitallySigned;

cpp/log/log_verifier.cc

@@ -10,6 +10,7 @@
 #include "proto/serializer.h"
 #include "util/util.h"
 
+using cert_trans::serialization::SerializeResult;
 using ct::LogEntry;
 using ct::MerkleAuditProof;
 using ct::SignedCertificateTimestamp;

cpp/log/logged_entry.cc

@@ -4,6 +4,7 @@
 #include "proto/serializer.h"
 #include "util/util.h"
 
+using cert_trans::serialization::SerializeResult;
 using ct::CertInfo;
 using ct::LogEntry;
 using ct::PreCert;

cpp/log/signer_verifier_test.cc

@@ -39,7 +39,7 @@ class SignerVerifierTest : public ::testing::Test {
 
   static string SerializedSignature(const DigitallySigned& signature) {
     string serialized_sig;
-    CHECK_EQ(SerializeResult::OK,
+    CHECK_EQ(cert_trans::serialization::SerializeResult::OK,
              Serializer::SerializeDigitallySigned(signature, &serialized_sig));
     return serialized_sig;
   }

cpp/log/test_signer.cc

@@ -470,7 +470,7 @@ void TestSigner::FillData(LoggedEntry* logged_cert) {
            Sha256Hasher::Sha256Digest(
                Serializer::LeafData(logged_cert->entry())));
   string serialized_leaf;
-  CHECK_EQ(SerializeResult::OK,
+  CHECK_EQ(cert_trans::serialization::SerializeResult::OK,
            Serializer::SerializeSCTMerkleTreeLeaf(logged_cert->sct(),
                                                   logged_cert->entry(),
                                                   &serialized_leaf));

cpp/merkletree/sparse_merkle_tree.cc

@@ -21,7 +21,9 @@ const vector<string>* GetNullHashes(const TreeHasher& hasher) {
   static unique_ptr<const vector<string>> null_hashes;
   if (!null_hashes) {
     vector<string> r{hasher.HashLeaf("")};
-    for (int i(1); i < hasher.DigestSize() * 8; ++i) {
+    const int end(hasher.DigestSize() * 8);
+    CHECK_LT(0, end);
+    for (int i(1); i < end; ++i) {
       r.emplace_back(hasher.HashChildren(r.back(), r.back()));
     }
     reverse(r.begin(), r.end());
@@ -135,7 +137,9 @@ string SparseMerkleTree::CalculateSubtreeHash(size_t depth, IndexType index) {
 
       case TreeNode::LEAF: {
         string ret(it->second.hash_);
-        for (int i(kDigestSizeBits - 1); i > depth; --i) {
+        const int64_t signed_depth(depth);
+        CHECK_LE(0, signed_depth);
+        for (int i(kDigestSizeBits - 1); i > signed_depth; --i) {
           if (PathBit(*(it->second.path_), i) == 0) {
             ret = treehasher_.HashChildren(ret, null_hashes_->at(i));
           } else {