Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PATCH] Minor: ssl: Build with new cryptographic library AWS-LC #1

Merged
merged 4 commits into from
Aug 31, 2023
Merged

[PATCH] Minor: ssl: Build with new cryptographic library AWS-LC #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
787760d
BUILD: ssl: Build with new cryptographic library AWS-LC
andrewhop Jul 6, 2023
3720a2e
MINOR: test: skip ssl_dh test when HAProxy is built with AWS-LC
andrewhop Aug 30, 2023
765578f
MINOR: ci: Add a weekly CI run with AWS-LC.
andrewhop Jul 13, 2023
7776c30
Add test workflow every 10 minutes
andrewhop Aug 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
329 changes: 175 additions & 154 deletions .github/matrix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,6 @@
import urllib.request
from os import environ

if len(sys.argv) == 2:
ref_name = sys.argv[1]
else:
print("Usage: {} <ref_name>".format(sys.argv[0]), file=sys.stderr)
sys.exit(1)

#
# this CI is used for both development and stable branches of HAProxy
#
Expand All @@ -29,34 +23,48 @@
# "haproxy-" - stable branches
# otherwise - development branch (i.e. "latest" ssl variants, "latest" github images)
#
print("Generating matrix for branch '{}'.".format(ref_name))


def clean_ssl(ssl):
return ssl.replace("_VERSION", "").lower()


@functools.lru_cache(5)
def determine_latest_openssl(ssl):
def get_all_github_tags(url):
headers = {}
if environ.get("GITHUB_TOKEN") is not None:
headers["Authorization"] = "token {}".format(environ.get("GITHUB_TOKEN"))
request = urllib.request.Request(
"https://api.github.com/repos/openssl/openssl/tags", headers=headers
)
request = urllib.request.Request(url, headers=headers)
try:
openssl_tags = urllib.request.urlopen(request)
tags = urllib.request.urlopen(request)
except:
return "OPENSSL_VERSION=failed_to_detect"
tags = json.loads(openssl_tags.read().decode("utf-8"))
return None
tags = json.loads(tags.read().decode("utf-8"))
return [tag['name'] for tag in tags]

@functools.lru_cache(5)
def determine_latest_openssl(ssl):
tags = get_all_github_tags("https://api.github.com/repos/openssl/openssl/tags")
if not tags:
return "OPENSSL_VERSION=failed_to_detect"
latest_tag = ""
for tag in tags:
name = tag["name"]
if "openssl-" in name:
if name > latest_tag:
latest_tag = name
if "openssl-" in tag:
if tag > latest_tag:
latest_tag = tag
return "OPENSSL_VERSION={}".format(latest_tag[8:])

def aws_lc_version_string_to_num(version_string):
return tuple(map(int, version_string[1:].split('.')))

def aws_lc_version_valid(version_string):
return re.match('^v[0-9]+(\.[0-9]+)*$', version_string)

@functools.lru_cache(5)
def determine_latest_aws_lc(ssl):
tags = get_all_github_tags("https://api.github.com/repos/aws/aws-lc/tags")
if not tags:
return "AWS_LC_VERSION=failed_to_detect"
valid_tags = list(filter(aws_lc_version_valid, tags))
latest_tag = max(valid_tags, key=aws_lc_version_string_to_num)
return "AWS_LC_VERSION={}".format(latest_tag[1:])

@functools.lru_cache(5)
def determine_latest_libressl(ssl):
Expand Down Expand Up @@ -86,159 +94,172 @@ def get_asan_flags(cc):
]


matrix = []

# Ubuntu

if "haproxy-" in ref_name:
os = "ubuntu-22.04" # stable branch
else:
os = "ubuntu-latest" # development branch

TARGET = "linux-glibc"
for CC in ["gcc", "clang"]:
matrix.append(
{
"name": "{}, {}, no features".format(os, CC),
"os": os,
"TARGET": TARGET,
"CC": CC,
"FLAGS": [],
}
)

matrix.append(
{
"name": "{}, {}, all features".format(os, CC),
"os": os,
"TARGET": TARGET,
"CC": CC,
"FLAGS": [
"USE_ZLIB=1",
"USE_OT=1",
"OT_INC=${HOME}/opt-ot/include",
"OT_LIB=${HOME}/opt-ot/lib",
"OT_RUNPATH=1",
"USE_PCRE=1",
"USE_PCRE_JIT=1",
"USE_LUA=1",
"USE_OPENSSL=1",
"USE_SYSTEMD=1",
"USE_WURFL=1",
"WURFL_INC=addons/wurfl/dummy",
"WURFL_LIB=addons/wurfl/dummy",
"USE_DEVICEATLAS=1",
"DEVICEATLAS_SRC=addons/deviceatlas/dummy",
"USE_PROMEX=1",
"USE_51DEGREES=1",
"51DEGREES_SRC=addons/51degrees/dummy/pattern",
],
}
)

# ASAN

matrix.append(
{
"name": "{}, {}, ASAN, all features".format(os, CC),
"os": os,
"TARGET": TARGET,
"CC": CC,
"FLAGS": get_asan_flags(CC)
+ [
"USE_ZLIB=1",
"USE_OT=1",
"OT_INC=${HOME}/opt-ot/include",
"OT_LIB=${HOME}/opt-ot/lib",
"OT_RUNPATH=1",
"USE_PCRE=1",
"USE_PCRE_JIT=1",
"USE_LUA=1",
"USE_OPENSSL=1",
"USE_SYSTEMD=1",
"USE_WURFL=1",
"WURFL_INC=addons/wurfl/dummy",
"WURFL_LIB=addons/wurfl/dummy",
"USE_DEVICEATLAS=1",
"DEVICEATLAS_SRC=addons/deviceatlas/dummy",
"USE_PROMEX=1",
"USE_51DEGREES=1",
"51DEGREES_SRC=addons/51degrees/dummy/pattern",
],
}
)

for compression in ["USE_ZLIB=1"]:

def main(ref_name):
print("Generating matrix for branch '{}'.".format(ref_name))
# Ubuntu
matrix = []
if "haproxy-" in ref_name:
os = "ubuntu-22.04" # stable branch
else:
os = "ubuntu-latest" # development branch

TARGET = "linux-glibc"
for CC in ["gcc", "clang"]:
matrix.append(
{
"name": "{}, {}, gz={}".format(os, CC, clean_compression(compression)),
"name": "{}, {}, no features".format(os, CC),
"os": os,
"TARGET": TARGET,
"CC": CC,
"FLAGS": [compression],
"FLAGS": [],
}
)

ssl_versions = [
"stock",
"OPENSSL_VERSION=1.0.2u",
"OPENSSL_VERSION=1.1.1s",
"QUICTLS=yes",
# "BORINGSSL=yes",
]

if "haproxy-" not in ref_name: # development branch
ssl_versions = ssl_versions + [
"OPENSSL_VERSION=latest",
"LIBRESSL_VERSION=latest",
]
matrix.append(
{
"name": "{}, {}, all features".format(os, CC),
"os": os,
"TARGET": TARGET,
"CC": CC,
"FLAGS": [
"USE_ZLIB=1",
"USE_OT=1",
"OT_INC=${HOME}/opt-ot/include",
"OT_LIB=${HOME}/opt-ot/lib",
"OT_RUNPATH=1",
"USE_PCRE=1",
"USE_PCRE_JIT=1",
"USE_LUA=1",
"USE_OPENSSL=1",
"USE_SYSTEMD=1",
"USE_WURFL=1",
"WURFL_INC=addons/wurfl/dummy",
"WURFL_LIB=addons/wurfl/dummy",
"USE_DEVICEATLAS=1",
"DEVICEATLAS_SRC=addons/deviceatlas/dummy",
"USE_PROMEX=1",
"USE_51DEGREES=1",
"51DEGREES_SRC=addons/51degrees/dummy/pattern",
],
}
)

for ssl in ssl_versions:
flags = ["USE_OPENSSL=1"]
if ssl == "BORINGSSL=yes" or ssl == "QUICTLS=yes" or "LIBRESSL" in ssl:
flags.append("USE_QUIC=1")
if ssl != "stock":
flags.append("SSL_LIB=${HOME}/opt/lib")
flags.append("SSL_INC=${HOME}/opt/include")
if "LIBRESSL" in ssl and "latest" in ssl:
ssl = determine_latest_libressl(ssl)
if "OPENSSL" in ssl and "latest" in ssl:
ssl = determine_latest_openssl(ssl)
# ASAN

matrix.append(
{
"name": "{}, {}, ssl={}".format(os, CC, clean_ssl(ssl)),
"name": "{}, {}, ASAN, all features".format(os, CC),
"os": os,
"TARGET": TARGET,
"CC": CC,
"ssl": ssl,
"FLAGS": flags,
"FLAGS": get_asan_flags(CC)
+ [
"USE_ZLIB=1",
"USE_OT=1",
"OT_INC=${HOME}/opt-ot/include",
"OT_LIB=${HOME}/opt-ot/lib",
"OT_RUNPATH=1",
"USE_PCRE=1",
"USE_PCRE_JIT=1",
"USE_LUA=1",
"USE_OPENSSL=1",
"USE_SYSTEMD=1",
"USE_WURFL=1",
"WURFL_INC=addons/wurfl/dummy",
"WURFL_LIB=addons/wurfl/dummy",
"USE_DEVICEATLAS=1",
"DEVICEATLAS_SRC=addons/deviceatlas/dummy",
"USE_PROMEX=1",
"USE_51DEGREES=1",
"51DEGREES_SRC=addons/51degrees/dummy/pattern",
],
}
)

# macOS
for compression in ["USE_ZLIB=1"]:
matrix.append(
{
"name": "{}, {}, gz={}".format(os, CC, clean_compression(compression)),
"os": os,
"TARGET": TARGET,
"CC": CC,
"FLAGS": [compression],
}
)

ssl_versions = [
"stock",
"OPENSSL_VERSION=1.0.2u",
"OPENSSL_VERSION=1.1.1s",
"QUICTLS=yes",
# "BORINGSSL=yes",
]

if "haproxy-" in ref_name:
os = "macos-12" # stable branch
else:
os = "macos-latest" # development branch
if "haproxy-" not in ref_name: # development branch
ssl_versions = ssl_versions + [
"OPENSSL_VERSION=latest",
"LIBRESSL_VERSION=latest",
'AWS_LC_VERSION=latest',
]

for ssl in ssl_versions:
flags = ["USE_OPENSSL=1"]
if ssl == "BORINGSSL=yes" or ssl == "QUICTLS=yes" or "LIBRESSL" in ssl:
flags.append("USE_QUIC=1")
if ssl != "stock":
flags.append("SSL_LIB=${HOME}/opt/lib")
flags.append("SSL_INC=${HOME}/opt/include")
if "LIBRESSL" in ssl and "latest" in ssl:
ssl = determine_latest_libressl(ssl)
if "OPENSSL" in ssl and "latest" in ssl:
ssl = determine_latest_openssl(ssl)
if "AWS_LC" in ssl and "latest" in ssl:
ssl = determine_latest_aws_lc(ssl)
flags.append("USE_OPENSSL_AWSLC=1")

matrix.append(
{
"name": "{}, {}, ssl={}".format(os, CC, clean_ssl(ssl)),
"os": os,
"TARGET": TARGET,
"CC": CC,
"ssl": ssl,
"FLAGS": flags,
}
)

# macOS

if "haproxy-" in ref_name:
os = "macos-12" # stable branch
else:
os = "macos-latest" # development branch

TARGET = "osx"
for CC in ["clang"]:
matrix.append(
{
"name": "{}, {}, no features".format(os, CC),
"os": os,
"TARGET": TARGET,
"CC": CC,
"FLAGS": [],
}
)

TARGET = "osx"
for CC in ["clang"]:
matrix.append(
{
"name": "{}, {}, no features".format(os, CC),
"os": os,
"TARGET": TARGET,
"CC": CC,
"FLAGS": [],
}
)
# Print matrix

# Print matrix
print(json.dumps(matrix, indent=4, sort_keys=True))

print(json.dumps(matrix, indent=4, sort_keys=True))
if environ.get("GITHUB_OUTPUT") is not None:
with open(environ.get("GITHUB_OUTPUT"), "a") as f:
print("matrix={}".format(json.dumps({"include": matrix})), file=f)

if environ.get("GITHUB_OUTPUT") is not None:
with open(environ.get("GITHUB_OUTPUT"), "a") as f:
print("matrix={}".format(json.dumps({"include": matrix})), file=f)
if __name__ == "__main__":
if len(sys.argv) == 2:
ref_name = sys.argv[1]
main(ref_name)
else:
print("Usage: {} <ref_name>".format(sys.argv[0]), file=sys.stderr)
sys.exit(1)
Loading