-
Notifications
You must be signed in to change notification settings - Fork 4
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
12 changed files
with
185 additions
and
92 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,42 +1,12 @@ | ||
| -- {{.TargetSchema}} | ||
| BEGIN; | ||
| CREATE DATABASE IF NOT EXISTS `{{.TargetSchema}}` DEFAULT CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_0900_ai_ci' ; | ||
| USE {{.TargetSchema}} ; | ||
|
|
||
| CREATE TABLE IF NOT EXISTS `urls` ( | ||
| `id` BIGINT UNSIGNED NOT NULL, | ||
| `url` VARCHAR(255) CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_0900_ai_ci' NOT NULL, | ||
| `parsed_url` VARCHAR(255) CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_0900_ai_ci' NOT NULL, | ||
| `fetch_time` BIGINT NOT NULL, | ||
| `expires` BIGINT NOT NULL, | ||
| `metadata` JSON NOT NULL, | ||
| `content_text` MEDIUMTEXT CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_0900_ai_ci' NULL, | ||
| PRIMARY KEY (`id`)); | ||
|
|
||
| CREATE TABLE IF NOT EXISTS `id_map` ( | ||
| `requested_id` BIGINT UNSIGNED NOT NULL, | ||
| `canonical_id` BIGINT UNSIGNED NOT NULL, | ||
| PRIMARY KEY (`requested_id`) | ||
| ); | ||
|
|
||
| -- Following two statements are added to support tracking headless | ||
| -- fetched state (or other alternate fetch methods) | ||
| -- The following cannot be executed idempotently | ||
| -- TODO: Goose migrations | ||
| ALTER TABLE urls ADD column fetch_method | ||
| INT UNSIGNED | ||
| NOT NULL DEFAULT 0; | ||
|
|
||
| CREATE INDEX fetch_method_expires_index ON urls ( | ||
| expires DESC, | ||
| fetch_method ASC | ||
| ); | ||
|
|
||
|
|
||
| CREATE ROLE IF NOT EXISTS scrape_app; | ||
| GRANT SELECT, INSERT, UPDATE, DELETE on {{.TargetSchema}}.* to scrape_app; | ||
| CREATE ROLE IF NOT EXISTS scrape_admin; | ||
| GRANT ALL ON {{.TargetSchema}}.* to scrape_admin; | ||
|
|
||
|
|
||
| USE {{.TargetSchema}} ; | ||
|
|
||
| COMMIT; | ||
| SET AUTOCOMMIT = 1; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| -- +goose Up | ||
| -- +goose StatementBegin | ||
|
|
||
| CREATE TABLE IF NOT EXISTS `urls` ( | ||
| `id` BIGINT UNSIGNED NOT NULL, | ||
| `url` VARCHAR(255) CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_0900_ai_ci' NOT NULL, | ||
| `parsed_url` VARCHAR(255) CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_0900_ai_ci' NOT NULL, | ||
| `fetch_time` BIGINT NOT NULL, | ||
| `fetch_method` INT UNSIGNED NOT NULL DEFAULT 0, | ||
| `expires` BIGINT NOT NULL, | ||
| `metadata` JSON NOT NULL, | ||
| `content_text` MEDIUMTEXT CHARACTER SET 'utf8mb4' COLLATE 'utf8mb4_0900_ai_ci' NULL, | ||
| PRIMARY KEY (`id`)); | ||
|
|
||
| CREATE TABLE IF NOT EXISTS `id_map` ( | ||
| `requested_id` BIGINT UNSIGNED NOT NULL, | ||
| `canonical_id` BIGINT UNSIGNED NOT NULL, | ||
| PRIMARY KEY (`requested_id`) | ||
| ); | ||
|
|
||
| CREATE INDEX fetch_method_expires_index ON urls ( | ||
| expires DESC, | ||
| fetch_method ASC | ||
| ); | ||
|
|
||
| -- +goose StatementEnd | ||
|
|
||
| -- +goose Down | ||
| -- +goose StatementBegin | ||
| USE `scrape_test`; | ||
| DROP TABLE IF EXISTS `urls`; | ||
| DROP TABLE IF EXISTS `id_map`; | ||
| -- +goose StatementEnd |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.