Merge branch 'release/0.1.6.7'

YukiOnishi1129 · Jun 21, 2024 · 75d1139 · 75d1139
2 parents 1fea701 + 3f8e291
commit 75d1139
Show file tree

Hide file tree

Showing 22 changed files with 918 additions and 25 deletions.
diff --git a/.github/workflows/batch_service_pr_check.yml b/.github/workflows/batch_service_pr_check.yml
@@ -22,6 +22,16 @@ jobs:
     strategy:
       matrix:
         go: ['1.22']
+
+    services:
+      postgres:
+        image: docker.io/postgres:16-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: root
+          POSTGRES_DB: password
+        ports:
+          - 5432:5432
 
     defaults:
       run:
@@ -45,6 +55,10 @@ jobs:
 
       - name: Go Test
         run: go test -v ./...
+        env:
+          POSTGRES_USER: root
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: postgres
 
       - name: Go Build
         run: go build -v ./...
diff --git a/batch-service/cmd/article-company-crawler/usecase/batch_crawl_article_contents.go b/batch-service/cmd/article-company-crawler/usecase/batch_crawl_article_contents.go
@@ -3,12 +3,13 @@ package usecase
 import (
 	"context"
 	"database/sql"
-	"github.com/YukiOnishi1129/techpicks/batch-service/domain"
-	"github.com/YukiOnishi1129/techpicks/batch-service/internal/crawler"
 	"log"
 	"strconv"
 	"time"
 
+	"github.com/YukiOnishi1129/techpicks/batch-service/domain"
+	"github.com/YukiOnishi1129/techpicks/batch-service/internal/crawler"
+
 	"github.com/YukiOnishi1129/techpicks/batch-service/entity"
 	"github.com/volatiletech/sqlboiler/v4/queries/qm"
 )
@@ -50,7 +51,7 @@ func (u *Usecase) BatchCrawlCompanyArticleContents(ctx context.Context) error {
 				log.Printf("【error begin transaction】: %s", err)
 				continue
 			}
-			res, err := crawler.ArticleContentsCrawler(ctx, tx, f, r)
+			res, err := crawler.ArticleContentsCrawler(ctx, tx, f, r, f.R.Platform.IsEng)
 			if err != nil && res.IsRollback {
 				log.Printf("【error rollback transaction】: %s", err)
 				err = tx.Rollback()

diff --git a/batch-service/cmd/article-site-and-summary-crawler/usecase/batch_crawl_article_contents.go b/batch-service/cmd/article-site-and-summary-crawler/usecase/batch_crawl_article_contents.go
@@ -3,12 +3,13 @@ package usecase
 import (
 	"context"
 	"database/sql"
-	"github.com/YukiOnishi1129/techpicks/batch-service/domain"
-	"github.com/YukiOnishi1129/techpicks/batch-service/internal/crawler"
 	"log"
 	"strconv"
 	"time"
 
+	"github.com/YukiOnishi1129/techpicks/batch-service/domain"
+	"github.com/YukiOnishi1129/techpicks/batch-service/internal/crawler"
+
 	"github.com/YukiOnishi1129/techpicks/batch-service/entity"
 	"github.com/volatiletech/sqlboiler/v4/queries/qm"
 )
@@ -53,7 +54,7 @@ func (u *Usecase) BatchCrawlSiteAndSummaryArticleContents(ctx context.Context) e
 				log.Printf("【error begin transaction】: %s", err)
 				continue
 			}
-			res, err := crawler.ArticleContentsCrawler(ctx, tx, f, r)
+			res, err := crawler.ArticleContentsCrawler(ctx, tx, f, r, f.R.Platform.IsEng)
 			if err != nil && res.IsRollback {
 				log.Printf("【error rollback transaction】: %s", err)
 				err = tx.Rollback()

diff --git a/batch-service/go.mod b/batch-service/go.mod
@@ -25,44 +25,83 @@ require (
 	cloud.google.com/go/iam v1.1.6 // indirect
 	cloud.google.com/go/longrunning v0.5.5 // indirect
 	cloud.google.com/go/storage v1.38.0 // indirect
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/PuerkitoBio/goquery v1.8.0 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/containerd/containerd v1.7.15 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
+	github.com/distribution/reference v0.5.0 // indirect
+	github.com/docker/docker v25.0.5+incompatible // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/gofrs/uuid v4.2.0+incompatible // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.2 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.6.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.16.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/machinebox/graphql v0.2.2 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
+	github.com/testcontainers/testcontainers-go v0.31.0 // indirect
+	github.com/testcontainers/testcontainers-go/modules/postgres v0.31.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/volatiletech/inflect v0.0.1 // indirect
 	github.com/volatiletech/randomize v0.0.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0 // indirect
-	go.opentelemetry.io/otel v1.23.0 // indirect
-	go.opentelemetry.io/otel/metric v1.23.0 // indirect
-	go.opentelemetry.io/otel/trace v1.23.0 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.13.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240304161311-37d4d3c04a78 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
 	google.golang.org/grpc v1.62.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )