Merge pull request #9 from NBISweden/push-static-files

Helm chart for landing pages

.dockerignore

@@ -1 +1,3 @@
-web/content/datasets/*
+web/content/datasets/*
+dev_utils/*
+nginx_proxy/*

.github/workflows/chart_publisher.yml

@@ -0,0 +1,29 @@
+name: Publish chart
+on:
+  push:
+    branches:
+      - main
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3.5.3
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3.5
+        with:
+          version: v3.4.0
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.5.0
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

.gitignore

@@ -1,8 +1,4 @@
-web/public/
-dev_utils/config.yaml
-web/static/img/m_p_b/
-web/static/img/m_p_c/
-web/static/img/m_p_p/
-web/static/img/m_r_f/
-web/content/datasets/*
+web/public/*
+web/content/datasets/test*
+web/data/*
 dev_utils/config.yaml

charts/landing-pages-chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/landing-pages-chart/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: landing-pages-chart
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

charts/landing-pages-chart/templates/ingress.yaml

@@ -0,0 +1,26 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/upstream-vhost: {{ .Values.s3Host}}
+    nginx.ingress.kubernetes.io/secure-backends: "true"
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    nginx.ingress.kubernetes.io/rewrite-log: "true"
+    nginx.ingress.kubernetes.io/rewrite-target: {{ .Values.bucketUrl }}/$1
+    nginx.ingress.kubernetes.io/use-regex: "true"
+    nginx.ingress.kubernetes.io/app-root: /index.html
+  name: my-ingress
+  namespace: {{ .Values.namespace }}
+spec:
+  ingressClassName: nginx
+  rules:
+  - http:
+      paths:
+      - backend:
+          service:
+            name: landing-pages-service
+            port:
+              number: 443
+        path: /(.*)
+        pathType: Prefix
+

charts/landing-pages-chart/templates/service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: landing-pages-service
+  namespace: {{ .Values.namespace }}
+spec:
+  externalName: {{ .Values.s3Host }}
+  sessionAffinity: None
+  type: ExternalName

charts/landing-pages-chart/values.yaml

@@ -0,0 +1,5 @@
+
+namespace:
+s3Host: 
+bucketUrl: 
+

download_metadata.go

@@ -16,7 +16,7 @@ func metadataDownloader(Metadataclient *MetadataBackend) {
 	var (
 		Bucket         = Metadataclient.Bucket // Download from this bucket
 		Prefix         = "datasets/"           // Using this key prefix
-		LocalDirectory = "web/content/"        // Into this directory
+		LocalDirectory = "web/data/"           // Into this directory
 	)
 	client := Metadataclient.Client
 

main.go

@@ -12,11 +12,14 @@ func main() {
 	Metadataclient := connectMetadatas3(mConf)
 	log.Infof("Connection to the bucket established")
 	metadataDownloader(Metadataclient)
+	markDownCreator()
 	cmd := exec.Command("hugo")
 	cmd.Dir = "./web/"
 	cmd.Run()
 	log.Infof("Hugo successfully built")
-	deConf := getDeploymentConfig()
-	connectDeployments3(deConf)
+	dConf := getDeploymentConfig()
+	DeploymenClient := connectDeployments3(dConf)
+	log.Infof("Connection to the bucket established")
+	test(DeploymenClient)
 
 }

markdown_writer.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func markDownCreator() {
+	// XML directory path (change this to your XML directory path)
+	xmlDirPath := "web/data/datasets/"
+
+	// Directory to save Markdown files
+	markdownDir := "web/content/datasets/"
+
+	// Create the Markdown directory if it doesn't exist
+	if err := os.MkdirAll(markdownDir, 0755); err != nil {
+		fmt.Println("Error creating directory:", err)
+		os.Exit(1)
+	}
+
+	// Walk through the XML directory
+	err := filepath.Walk(xmlDirPath, func(xmlFilePath string, info os.FileInfo, err error) error {
+		if err != nil {
+			fmt.Printf("Error accessing file %s: %v\n", xmlFilePath, err)
+			return nil
+		}
+		// Skip directories
+		if info.IsDir() {
+			return nil
+		}
+
+		// Read XML file name
+		xmlFileName := filepath.Base(xmlFilePath)
+
+		// Remove file extension
+		fileNameWithoutExt := strings.TrimSuffix(xmlFileName, filepath.Ext(xmlFileName))
+
+		// Markdown content
+		markdownContent := fmt.Sprintf(`
+---
+---
+
+{{< datafetch variable="%s" >}}
+
+
+Filename of the associated XML file: %s
+`, fileNameWithoutExt, xmlFileName)
+
+		// Create Markdown file
+		mdFileName := filepath.Join(markdownDir, fileNameWithoutExt+".md")
+		mdFile, err := os.Create(mdFileName)
+		if err != nil {
+			fmt.Printf("Error creating Markdown file %s: %v\n", mdFileName, err)
+			return nil
+		}
+		defer mdFile.Close()
+
+		// Write Markdown content to the file
+		_, err = mdFile.WriteString(markdownContent)
+		if err != nil {
+			fmt.Printf("Error writing to Markdown file %s: %v\n", mdFileName, err)
+			return nil
+		}
+
+		fmt.Printf("Markdown file %s created successfully!\n", mdFileName)
+
+		return nil
+	})
+
+	if err != nil {
+		fmt.Println("Error walking through directory:", err)
+		os.Exit(1)
+	}
+}

s3.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -23,16 +22,11 @@ type DeploymentBackend struct {
 }
 
 func connectMetadatas3(mConf MetadataS3Config) *MetadataBackend {
-	httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) {
-		if tr.TLSClientConfig == nil {
-			tr.TLSClientConfig = &tls.Config{}
-		}
-		tr.TLSClientConfig.MinVersion = tls.VersionTLS13
-	})
 
 	cfg, err := config.LoadDefaultConfig(context.TODO(),
 		config.WithRegion("auto"),
-		config.WithHTTPClient(httpClient),
+		config.WithHTTPClient(&http.Client{Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}),
 		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(mConf.AccessKey, mConf.SecretKey, "")),
 		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
 			func(service, region string, options ...interface{}) (aws.Endpoint, error) {
@@ -61,16 +55,11 @@ func connectMetadatas3(mConf MetadataS3Config) *MetadataBackend {
 }
 
 func connectDeployments3(dConf DeployS3Config) *DeploymentBackend {
-	httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) {
-		if tr.TLSClientConfig == nil {
-			tr.TLSClientConfig = &tls.Config{}
-		}
-		tr.TLSClientConfig.MinVersion = tls.VersionTLS13
-	})
 
 	cfg, err := config.LoadDefaultConfig(context.TODO(),
 		config.WithRegion("auto"),
-		config.WithHTTPClient(httpClient),
+		config.WithHTTPClient(&http.Client{Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}),
 		config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(dConf.AccessKey, dConf.SecretKey, "")),
 		config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc(
 			func(service, region string, options ...interface{}) (aws.Endpoint, error) {

static_files_uploader.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"context"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+func test(DeploymenClient *DeploymentBackend) {
+	var (
+		localPath = "web/public/"
+		bucket    = DeploymenClient.Bucket
+	)
+
+	walker := make(fileWalk)
+	go func() {
+		// Gather the files to upload by walking the path recursively
+		if err := filepath.Walk(localPath, walker.Walk); err != nil {
+			log.Fatalln("Walk failed:", err)
+		}
+		close(walker)
+	}()
+
+	// For each file found walking, upload it to Amazon S3
+	for path := range walker {
+		rel, err := filepath.Rel(localPath, path)
+		if err != nil {
+			log.Fatalln("Unable to get relative path:", path, err)
+		}
+		file, err := os.Open(path)
+		if err != nil {
+			log.Println("Failed opening file", path, err)
+			continue
+		}
+		buf := make([]byte, 512)
+		_, err = file.Read(buf)
+		contentType := http.DetectContentType(buf)
+		ext := filepath.Ext(path)
+		if ext == ".css" {
+			contentType = "text/css;"
+		}
+
+		if err != nil {
+			log.Fatalln("File bytes empty", path, err)
+		}
+
+		client := DeploymenClient.Client
+		uploader := manager.NewUploader(client)
+		result, err := uploader.Upload(context.TODO(), &s3.PutObjectInput{
+			Bucket:      &bucket,
+			Key:         aws.String(rel),
+			Body:        file,
+			ContentType: aws.String(contentType),
+		})
+		println(contentType)
+		file.Close()
+		if err != nil {
+			log.Fatalln("Failed to upload", path, err)
+		}
+		log.Infoln("Uploaded", path, result.Location)
+	}
+}
+
+type fileWalk chan string
+
+func (f fileWalk) Walk(path string, info os.FileInfo, err error) error {
+	if err != nil {
+		return err
+	}
+	if !info.IsDir() {
+		f <- path
+	}
+	return nil
+}

web/README.md

@@ -0,0 +1,2 @@
+# hugo.bp.datasets.nbis.se
+Bigpicture landing pages and data description pages using HUGO