hack: update format of rhcos build metadata

This splits the RHCOS build metadata into architecture-specific files.
This will allow the metadata to contain information about bootimages of
multiple architectures. In order to preserve backward compatibility
(there are a few users, including certain CI jobs, that pull rhcos.json
from GitHub directly), I've opted to use separate files for each
architecture. Normally, we could have just symlinked the legacy metadata
file, but when hosted on raw.githubcontent.com, the symlinks aren't
followed.

When updating the RHCOS bootimages, this script will need to be run once
for each architecture that is being updated.

The build metadata was resynced with the following:

./hack/update-rhcos-bootimage.py https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.2/42.80.20191002.0/meta.json amd64

data/data/rhcos-amd64.json

@@ -0,0 +1,137 @@
+{
+    "amis": {
+        "ap-northeast-1": {
+            "hvm": "ami-0426ca3481a088c7b"
+        },
+        "ap-northeast-2": {
+            "hvm": "ami-014514ae47679721b"
+        },
+        "ap-south-1": {
+            "hvm": "ami-0bd772ba746948d9a"
+        },
+        "ap-southeast-1": {
+            "hvm": "ami-0d76ac0ebaac29e40"
+        },
+        "ap-southeast-2": {
+            "hvm": "ami-0391e92574fb09e08"
+        },
+        "ca-central-1": {
+            "hvm": "ami-04419691da69850cf"
+        },
+        "eu-central-1": {
+            "hvm": "ami-092b69120ecf915ed"
+        },
+        "eu-north-1": {
+            "hvm": "ami-0175e9c9d258cc11d"
+        },
+        "eu-west-1": {
+            "hvm": "ami-04370efd78434697b"
+        },
+        "eu-west-2": {
+            "hvm": "ami-00c74e593125e0096"
+        },
+        "eu-west-3": {
+            "hvm": "ami-058ad17da14ff4d0d"
+        },
+        "me-south-1": {
+            "hvm": "ami-04ab7423e63117ec0"
+        },
+        "sa-east-1": {
+            "hvm": "ami-03f6b71e93e630dab"
+        },
+        "us-east-1": {
+            "hvm": "ami-01e7fdcb66157b224"
+        },
+        "us-east-2": {
+            "hvm": "ami-0bc59aaa7363b805d"
+        },
+        "us-west-1": {
+            "hvm": "ami-0ba912f53c1fdcdf0"
+        },
+        "us-west-2": {
+            "hvm": "ami-08e10b201e19fd5e7"
+        }
+    },
+    "azure": {
+        "image": "rhcos-42.80.20191002.0.vhd",
+        "url": "https://rhcos.blob.core.windows.net/imagebucket/rhcos-42.80.20191002.0.vhd"
+    },
+    "baseURI": "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.2/42.80.20191002.0/",
+    "buildid": "42.80.20191002.0",
+    "gcp": {
+        "image": "rhcos-42-80-20191002-0",
+        "url": "https://storage.googleapis.com/rhcos/rhcos/42.80.20191002.0.tar.gz"
+    },
+    "images": {
+        "aws": {
+            "path": "rhcos-42.80.20191002.0-aws.vmdk",
+            "sha256": "f5dbf8b4721609214101cda353789557de28caffd496b90c62ace46b9f5e0b9f",
+            "size": 710720458,
+            "uncompressed-sha256": "63545c92997f487217541e128eee4aa81ab068840e2f988acbe5848ff711388f",
+            "uncompressed-size": 725895168
+        },
+        "azure": {
+            "path": "rhcos-42.80.20191002.0.vhd",
+            "sha256": "7336694bcb936b7c95912ba49f9432adbe547837574ccd3e7c4b97a969c95a8c",
+            "size": 698788031,
+            "uncompressed-sha256": "1d8c227ec4f22822dfb3a132a764f2e64616e2cb9dd34290e87ff9b805a467fd",
+            "uncompressed-size": 1900518912
+        },
+        "gcp": {
+            "path": "rhcos-42.80.20191002.0-gcp.tar",
+            "sha256": "15434ee2b7d32368315519357dd0a036f67c7f9e9b987027512099b6b6d66aaf",
+            "size": 698392984
+        },
+        "initramfs": {
+            "path": "rhcos-42.80.20191002.0-installer-initramfs.img",
+            "sha256": "a975a0e5ebdbbd186a94fd2a2fdb122c88b228fd5a44c3118cf0b29f52f58995"
+        },
+        "iso": {
+            "path": "rhcos-42.80.20191002.0-installer.iso",
+            "sha256": "649cdcf265bf39c0386331ec9b100da7c4800eae6c5aab4a8c28514b6efc5289"
+        },
+        "kernel": {
+            "path": "rhcos-42.80.20191002.0-installer-kernel",
+            "sha256": "a31a100ad4ad033240ad8547f55a803542f7a6212adb7df1d0fb4618383b9729"
+        },
+        "metal-bios": {
+            "path": "rhcos-42.80.20191002.0-metal-bios.raw.gz",
+            "sha256": "4de137e8e3e07f57fa6295b4f95be503f26835a782f9778607e12551669ee6bc",
+            "size": 700157452,
+            "uncompressed-sha256": "1ee6d0f9d931396ba089489646ad4a0046014cd59e08f38918ebe500b34e498a",
+            "uncompressed-size": 3182428160
+        },
+        "metal-uefi": {
+            "path": "rhcos-42.80.20191002.0-metal-uefi.raw.gz",
+            "sha256": "ee6c25c0eeb44e6016ed8604e2c302c7c8c940ef372277f62b23ccdad5a2225e",
+            "size": 699608952,
+            "uncompressed-sha256": "c1f2d7e12d5c2f96cd2d373111831e4aae85a6c2a1a9ce9ffb987678a11c39bd",
+            "uncompressed-size": 3182428160
+        },
+        "openstack": {
+            "path": "rhcos-42.80.20191002.0-openstack.qcow2",
+            "sha256": "b9225f67ae16ef38625c9669d556324d6422b29b5f6bfbd87e960062bfb37f70",
+            "size": 699870352,
+            "uncompressed-sha256": "414a90ca4fca60a7d25dee02f63d51ad5afc1a614e6a76135e26a1ec62ef40d1",
+            "uncompressed-size": 1911160832
+        },
+        "qemu": {
+            "path": "rhcos-42.80.20191002.0-qemu.qcow2",
+            "sha256": "ff8a785d749f5ef771f966e11c498e3eb214f9f82856e5f2d62e81f2f89062c9",
+            "size": 699873960,
+            "uncompressed-sha256": "3479ae1cdedc4dd983fba0e356ef2649d2c74bf97b71d818b1c4c31d16d6674b",
+            "uncompressed-size": 1911095296
+        },
+        "vmware": {
+            "path": "rhcos-42.80.20191002.0-vmware.ova",
+            "sha256": "91a729bde4512dd5d5a7ab060c482aea3ee48d56c7f5d8ce51f8596c8b7827bf",
+            "size": 725903360
+        }
+    },
+    "oscontainer": {
+        "digest": "sha256:cc71fbd134f063d9fc0ccc78933b89c8dd2b1418b7a7b85bb70de87bc80486d7",
+        "image": "quay.io/openshift-release-dev/ocp-v4.0-art-dev"
+    },
+    "ostree-commit": "3f772530680edb70ded914f213c07cfa740f1d0ca351941af8e7e5b3e40b8451",
+    "ostree-version": "42.80.20191002.0"
+}

data/data/rhcos.json

@@ -33,6 +33,9 @@
         "eu-west-3": {
             "hvm": "ami-058ad17da14ff4d0d"
         },
+        "me-south-1": {
+            "hvm": "ami-04ab7423e63117ec0"
+        },
         "sa-east-1": {
             "hvm": "ami-03f6b71e93e630dab"
         },

hack/update-rhcos-bootimage.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Usage: ./hack/update-rhcos-bootimage.py https://releases-art-rhcos.svc.ci.openshift.org/storage/releases/ootpa/410.8.20190401.0/meta.json
+# Usage: ./hack/update-rhcos-bootimage.py https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases/rhcos-4.2/42.80.20191002.0/meta.json amd64
 import codecs,os,sys,json,argparse
 import urllib.parse
 import urllib.request
@@ -8,15 +8,16 @@
 # builds.  Do not try to e.g. point to RHT-internal endpoints.
 RHCOS_RELEASES_APP = 'https://releases-art-rhcos.svc.ci.openshift.org'
 
-dn = os.path.abspath(os.path.dirname(sys.argv[0]))
-
 parser = argparse.ArgumentParser()
 parser.add_argument("meta", action='store')
+parser.add_argument("arch", action='store', choices=['amd64'])
 args = parser.parse_args()
 
 if not args.meta.startswith(RHCOS_RELEASES_APP):
     raise SystemExit("URL must start with: " + RHCOS_RELEASES_APP)
 
+metadata_dir = os.path.join(os.path.dirname(sys.argv[0]), "../data/data")
+
 with urllib.request.urlopen(args.meta) as f:
     string_f = codecs.getreader('utf-8')(f)  # support for Python < 3.6
     meta = json.load(string_f)
@@ -32,5 +33,14 @@
     for entry in meta['amis']
 }
 newmeta['baseURI'] = urllib.parse.urljoin(args.meta, '.')
-with open(os.path.join(dn, "../data/data/rhcos.json"), 'w') as f:
+
+with open(os.path.join(metadata_dir, f"rhcos-{args.arch}.json"), 'w') as f:
     json.dump(newmeta, f, sort_keys=True, indent=4)
+
+# Continue to populate the legacy metadata file because there are still
+# processes consuming this file directly. This normally could just be a symlink
+# but some of these processes reference raw.githubusercontent.com which doesn't
+# follow symlinks.
+if args.arch == 'amd64':
+    with open(os.path.join(metadata_dir, "rhcos.json"), 'w') as f:
+        json.dump(newmeta, f, sort_keys=True, indent=4)