-
Notifications
You must be signed in to change notification settings - Fork 111
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
9e23123
commit 0934d73
Showing
25 changed files
with
375 additions
and
250 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
NODE_ENV='dev' | ||
|
||
AWS_ACCESS_KEY_ID= | ||
AWS_SECRET_ACCESS_KEY= | ||
AWS_REGION= | ||
AWS_BUCKET_RAW= | ||
AWS_BUCKET_INDEXED= | ||
|
||
DDEX_KEY= | ||
DDEX_SECRET= | ||
|
||
# use stage optimizely | ||
OPTIMIZELY_SDK_KEY='MX4fYBgANQetvmBXGpuxzF' |
7 changes: 7 additions & 0 deletions
7
packages/ddex/webapp/server/.env.stage → packages/ddex/.env.stage
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,11 @@ | ||
NODE_ENV='stage' | ||
|
||
AWS_ACCESS_KEY_ID= | ||
AWS_SECRET_ACCESS_KEY= | ||
AWS_REGION='us-west-2' | ||
AWS_BUCKET_RAW='ddex-dev-audius-raw' | ||
AWS_BUCKET_INDEXED='ddex-dev-audius-indexed' | ||
|
||
DDEX_KEY='49d5e13d355709b615b7cce7369174fb240b6b39' | ||
DDEX_SECRET='2b2c2b90d9a489234ae629a5284de84fb0633306257f17667aaebf2345d92152' | ||
OPTIMIZELY_SDK_KEY='MX4fYBgANQetvmBXGpuxzF' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,33 +1,55 @@ | ||
package main | ||
|
||
import ( | ||
"context" | ||
"flag" | ||
"fmt" | ||
"ingester/crawler" | ||
"ingester/indexer" | ||
"ingester/parser" | ||
"log" | ||
"os" | ||
"os/signal" | ||
"syscall" | ||
|
||
"github.com/joho/godotenv" | ||
) | ||
|
||
func main() { | ||
service := flag.String("service", "", "Specify the service to run: crawler, indexer, or parser") | ||
flag.Parse() | ||
|
||
ctx, cancel := context.WithCancel(context.Background()) | ||
defer cancel() | ||
|
||
go func() { | ||
sigChan := make(chan os.Signal, 1) | ||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) | ||
sig := <-sigChan | ||
log.Printf("Received signal: %v, shutting down...\n", sig) | ||
cancel() | ||
}() | ||
|
||
err := godotenv.Load("../.env") | ||
if err != nil { | ||
if os.IsNotExist(err) { | ||
log.Println("No .env file found, proceeding with existing environment variables") | ||
} else { | ||
log.Println("Error loading .env file:", err) | ||
} | ||
} | ||
|
||
switch *service { | ||
case "crawler": | ||
crawler.Run() | ||
go crawler.Run(ctx) | ||
case "indexer": | ||
indexer.Run() | ||
go indexer.Run(ctx) | ||
case "parser": | ||
parser.Run() | ||
go parser.Run(ctx) | ||
default: | ||
fmt.Println("Unknown service: " + *service) | ||
// sleep | ||
sigChan := make(chan os.Signal, 1) | ||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) | ||
sig := <-sigChan | ||
fmt.Printf("Received signal: %v, shutting down...\n", sig) | ||
} | ||
|
||
<-ctx.Done() // Wait until the context is canceled | ||
log.Println("Service stopped") | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,54 @@ | ||
package common | ||
|
||
import ( | ||
"context" | ||
"log" | ||
"os" | ||
"time" | ||
|
||
"github.com/aws/aws-sdk-go/aws" | ||
"github.com/aws/aws-sdk-go/aws/credentials" | ||
"github.com/aws/aws-sdk-go/aws/session" | ||
"github.com/aws/aws-sdk-go/service/s3" | ||
"go.mongodb.org/mongo-driver/mongo" | ||
"go.mongodb.org/mongo-driver/mongo/options" | ||
) | ||
|
||
func InitMongoClient(ctx context.Context) *mongo.Client { | ||
mongoUrl := os.Getenv("DDEX_MONGODB_URL") | ||
if mongoUrl == "" { | ||
mongoUrl = "mongodb://mongo:mongo@localhost:27017/ddex?authSource=admin&replicaSet=rs0" | ||
} | ||
client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoUrl)) | ||
if err != nil { | ||
panic(err) | ||
} | ||
log.Println("Connected to mongo") | ||
return client | ||
} | ||
|
||
func InitS3Client() *s3.S3 { | ||
awsRegion := MustGetenv("AWS_REGION") | ||
awsKey := MustGetenv("AWS_ACCESS_KEY_ID") | ||
awsSecret := MustGetenv("AWS_SECRET_ACCESS_KEY") | ||
sess, err := session.NewSession(&aws.Config{ | ||
Region: aws.String(awsRegion), | ||
Credentials: credentials.NewStaticCredentials(awsKey, awsSecret, ""), | ||
}) | ||
if err != nil { | ||
panic(err) | ||
} | ||
log.Println("Connected to s3") | ||
return s3.New(sess) | ||
} | ||
|
||
func MustGetenv(key string) string { | ||
val := os.Getenv(key) | ||
if val == "" { | ||
log.Println("Missing required env variable: ", key, " sleeping ...") | ||
// If config is incorrect, sleep a bit to prevent container from restarting constantly | ||
time.Sleep(time.Hour) | ||
log.Fatal("Missing required env variable: ", key) | ||
} | ||
return val | ||
} |
Oops, something went wrong.