From 45f7fc8606c18a4c5fd78b63a9e159c42f62ad50 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 15:53:52 +0800 Subject: [PATCH 01/37] Add markdown lint action --- .github/workflows/build-deploy-docs.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index 81d0713ed..9331af29c 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -15,6 +15,13 @@ jobs: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 + - name: Markdown lint + uses: DavidAnson/markdownlint-cli2-action@v14 + with: + globs: | + docs/src/*.md + README.md + - name: Setup mdBook uses: peaceiris/actions-mdbook@v1 with: From b0c1ff1246072e90abf5c03f2225914aa214bcab Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 16:15:54 +0800 Subject: [PATCH 02/37] Add custom markdown config file --- .github/files/.markdownlint.jsonc | 8 ++++++++ .github/workflows/build-deploy-docs.yml | 4 +--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 .github/files/.markdownlint.jsonc diff --git a/.github/files/.markdownlint.jsonc b/.github/files/.markdownlint.jsonc new file mode 100644 index 000000000..6e0206709 --- /dev/null +++ b/.github/files/.markdownlint.jsonc @@ -0,0 +1,8 @@ +{ + "config": { + "default": true, + "relative-links": true + }, + "globs": ["docs/src/*.md"], + "customRules": ["markdownlint-rule-relative-links"] +} \ No newline at end of file diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index 9331af29c..949ea941a 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -18,9 +18,7 @@ jobs: - name: Markdown lint uses: DavidAnson/markdownlint-cli2-action@v14 with: - globs: | - docs/src/*.md - README.md + config: './.github/files/.markdownlint.jsonc' - name: Setup mdBook uses: peaceiris/actions-mdbook@v1 From 6eb61f57fcddd586ab330cf628cfe8fbc1974183 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 16:37:31 +0800 Subject: [PATCH 03/37] Update globs expressions --- .github/files/.markdownlint.jsonc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/files/.markdownlint.jsonc b/.github/files/.markdownlint.jsonc index 6e0206709..46794377b 100644 --- a/.github/files/.markdownlint.jsonc +++ b/.github/files/.markdownlint.jsonc @@ -3,6 +3,10 @@ "default": true, "relative-links": true }, - "globs": ["docs/src/*.md"], + // globs expression. See https://github.com/DavidAnson/markdownlint-cli2#command-line + "globs": [ + "**/*.md", + "#CHANGELOG.md" + ], "customRules": ["markdownlint-rule-relative-links"] -} \ No newline at end of file +} From f2f8d15cafbca5fd066b554b7e2cdfb648b8e3bd Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 16:47:18 +0800 Subject: [PATCH 04/37] Ignore the linting of `CHANGELOG.md` --- .github/files/.markdownlint.jsonc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/files/.markdownlint.jsonc b/.github/files/.markdownlint.jsonc index 46794377b..f3ae03cdd 100644 --- a/.github/files/.markdownlint.jsonc +++ b/.github/files/.markdownlint.jsonc @@ -5,8 +5,10 @@ }, // globs expression. See https://github.com/DavidAnson/markdownlint-cli2#command-line "globs": [ - "**/*.md", - "#CHANGELOG.md" + "**/*.md" + ], + "ignores":[ + ":CHANGELOG.md" ], "customRules": ["markdownlint-rule-relative-links"] } From f9e912501d650d9e1b1131b8923b815f0c9c914e Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 17:19:33 +0800 Subject: [PATCH 05/37] Update --- .../{.markdownlint.jsonc => .markdownlint-cli2.jsonc} | 11 +++++------ .github/workflows/build-deploy-docs.yml | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) rename .github/files/{.markdownlint.jsonc => .markdownlint-cli2.jsonc} (58%) diff --git a/.github/files/.markdownlint.jsonc b/.github/files/.markdownlint-cli2.jsonc similarity index 58% rename from .github/files/.markdownlint.jsonc rename to .github/files/.markdownlint-cli2.jsonc index f3ae03cdd..f5eea1eec 100644 --- a/.github/files/.markdownlint.jsonc +++ b/.github/files/.markdownlint-cli2.jsonc @@ -5,10 +5,9 @@ }, // globs expression. See https://github.com/DavidAnson/markdownlint-cli2#command-line "globs": [ - "**/*.md" - ], - "ignores":[ - ":CHANGELOG.md" - ], - "customRules": ["markdownlint-rule-relative-links"] + "README.md", + "!CHANGELOG.md", + "docs/src/*.md" + ] + // "customRules": ["markdownlint-rule-relative-links"] } diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index 949ea941a..7f8f725e1 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -18,7 +18,7 @@ jobs: - name: Markdown lint uses: DavidAnson/markdownlint-cli2-action@v14 with: - config: './.github/files/.markdownlint.jsonc' + config: './.github/files/.markdownlint-cli2.jsonc' - name: Setup mdBook uses: peaceiris/actions-mdbook@v1 From a1cd714088a9304056922a3e43114af9d672085b Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 17:44:19 +0800 Subject: [PATCH 06/37] Eliminate `MD022/blanks-around-headings` warnings --- docs/src/mbtiles-copy.md | 3 +++ docs/src/mbtiles-meta.md | 4 ++++ docs/src/mbtiles-schema.md | 4 ++++ docs/src/mbtiles-validation.md | 3 +++ 4 files changed, 14 insertions(+) diff --git a/docs/src/mbtiles-copy.md b/docs/src/mbtiles-copy.md index e87cf6757..0e8098edb 100644 --- a/docs/src/mbtiles-copy.md +++ b/docs/src/mbtiles-copy.md @@ -1,6 +1,7 @@ # Copying, Diffing, and Patching MBTiles ## `mbtiles copy` + Copy command copies an mbtiles file, optionally filtering its content by zoom levels. ```shell @@ -16,6 +17,7 @@ mbtiles copy normalized.mbtiles dst.mbtiles \ ``` ## `mbtiles copy --diff-with-file` + Copy command can also be used to compare two mbtiles files and generate a delta (diff) file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset. The delta file will contain all tiles that are different between the two files (modifications, insertions, and deletions as `NULL` values), for both the tile and metadata tables. There is one exception: `agg_tiles_hash` metadata value will be renamed to `agg_tiles_hash_in_diff`, and a new `agg_tiles_hash` will be generated for the diff file itself. This is done to avoid confusion when applying the diff file to the original file, as the `agg_tiles_hash` value will be different after the diff is applied. The `apply-diff` command will automatically rename the `agg_tiles_hash_in_diff` value back to `agg_tiles_hash` when applying the diff. @@ -45,6 +47,7 @@ mbtiles apply_diff src_file.mbtiles diff_file.mbtiles ``` #### Applying diff with SQLite + Another way to apply the diff is to use the `sqlite3` command line tool directly. This SQL will delete all tiles from `src_file.mbtiles` that are set to `NULL` in `diff_file.mbtiles`, and then insert or update all new tiles from `diff_file.mbtiles` into `src_file.mbtiles`, where both files are of `flat` type. The name of the diff file is passed as a query parameter to the sqlite3 command line tool, and then used in the SQL statements. Note that this does not update the `agg_tiles_hash` metadata value, so it will be incorrect after the diff is applied. ```shell diff --git a/docs/src/mbtiles-meta.md b/docs/src/mbtiles-meta.md index 8aca2fb55..ac1d0fb76 100644 --- a/docs/src/mbtiles-meta.md +++ b/docs/src/mbtiles-meta.md @@ -1,6 +1,7 @@ # MBTiles information and metadata ## summary + Use `mbtiles summary` to get a summary of the contents of an MBTiles file. The command will print a table with the number of tiles per zoom level, the size of the smallest and largest tiles, and the average size of tiles at each zoom level. The command will also print the bounding box of the covered area per zoom level. ```shell @@ -22,6 +23,7 @@ Page count: 12 ``` ## meta-all + Print all metadata values to stdout, as well as the results of tile detection. The format of the values printed is not stable, and should only be used for visual inspection. ```shell @@ -29,6 +31,7 @@ mbtiles meta-all my_file.mbtiles ``` ## meta-get + Retrieve raw metadata value by its name. The value is printed to stdout without any modifications. For example, to get the `description` value from an mbtiles file: ```shell @@ -36,6 +39,7 @@ mbtiles meta-get my_file.mbtiles description ``` ## meta-set + Set metadata value by its name, or delete the key if no value is supplied. For example, to set the `description` value to `A vector tile dataset`: ```shell diff --git a/docs/src/mbtiles-schema.md b/docs/src/mbtiles-schema.md index 188196b48..561c81d7e 100644 --- a/docs/src/mbtiles-schema.md +++ b/docs/src/mbtiles-schema.md @@ -1,7 +1,9 @@ # MBTiles Schemas + The `mbtiles` tool builds on top of the original [MBTiles specification](https://github.com/mapbox/mbtiles-spec#readme) by specifying three different kinds of schema for `tiles` data: `flat`, `flat-with-hash`, and `normalized`. The `mbtiles` tool can convert between these schemas, and can also generate a diff between two files of any schemas, as well as merge multiple schema files into one file. ## flat + Flat schema is the closest to the original MBTiles specification. It stores all tiles in a single table. This schema is the most efficient when the tileset contains no duplicate tiles. ```sql, ignore @@ -16,6 +18,7 @@ CREATE UNIQUE INDEX tile_index on tiles ( ``` ## flat-with-hash + Similar to the `flat` schema, but also includes a `tile_hash` column that contains a hash value of the `tile_data` column. Use this schema when the tileset has no duplicate tiles, but you still want to be able to validate the content of each tile individually. ```sql, ignore @@ -35,6 +38,7 @@ CREATE VIEW tiles AS ``` ## normalized + Normalized schema is the most efficient when the tileset contains duplicate tiles. It stores all tile blobs in the `images` table, and stores the tile Z,X,Y coordinates in a `map` table. The `map` table contains a `tile_id` column that is a foreign key to the `images` table. The `tile_id` column is a hash of the `tile_data` column, making it possible to both validate each individual tile like in the `flat-with-hash` schema, and also to optimize storage by storing each unique tile only once. ```sql, ignore diff --git a/docs/src/mbtiles-validation.md b/docs/src/mbtiles-validation.md index 3c049d8e3..bd5486c5e 100644 --- a/docs/src/mbtiles-validation.md +++ b/docs/src/mbtiles-validation.md @@ -7,12 +7,15 @@ mbtiles validate src_file.mbtiles ``` ## SQLite Integrity check + The `validate` command will run `PRAGMA integrity_check` on the file, and will fail if the result is not `ok`. The `--integrity-check` flag can be used to disable this check, or to make it more thorough with `full` value. Default is `quick`. ## Schema check + The `validate` command will verify that the `tiles` table/view exists, and that it has the expected columns and indexes. It will also verify that the `metadata` table/view exists, and that it has the expected columns and indexes. ## Per-tile validation + If the `.mbtiles` file uses [flat_with_hash](mbtiles-schema.md#flat-with-hash) or [normalized](mbtiles-schema.md#normalized) schema, the `validate` command will verify that the MD5 hash of the `tile_data` column matches the `tile_hash` or `tile_id` columns (depending on the schema). A typical Normalized schema generated by tools like [tilelive-copy](https://github.com/mapbox/TileLive#bintilelive-copy) use MD5 hash in the `tile_id` column. The Martin's `mbtiles` tool can use this hash to verify the content of each tile. We also define a new [flat-with-hash](mbtiles-schema.md#flat-with-hash) schema that stores the hash and tile data in the same table, allowing per-tile validation without the multiple table layout. From da85d68f3a1699b08c3d541017e7cc3468e5e42f Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 20:44:28 +0800 Subject: [PATCH 07/37] Eliminate `MD022/blanks-around-headings` warnings --- README.md | 5 +++++ docs/src/using.md | 2 ++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index f3ce97e18..11346400b 100755 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Additionally, there are [several tools](https://maplibre.org/martin/tools.html) See [Martin book](https://maplibre.org/martin/) for complete documentation. ## Installation + _See [installation instructions](https://maplibre.org/martin/installation.html) in the Martin book._ **Prerequisites:** If using Martin with PostgreSQL database, you must install PostGIS with at least v3.0+, v3.1+ recommended. @@ -45,6 +46,7 @@ brew install martin ``` ## Running Martin Service + _See [running instructions](https://maplibre.org/martin/run.html) in the Martin book._ Martin supports any number of PostgreSQL/PostGIS database connections with [geospatial-enabled](https://postgis.net/docs/using_postgis_dbmanagement.html#geometry_columns) tables and tile-producing SQL functions, as well as [PMTile](https://protomaps.com/blog/pmtiles-v3-whats-new) and [MBTile](https://github.com/mapbox/mbtiles-spec) files as tile sources. @@ -72,6 +74,7 @@ martin --config config.yaml ``` #### Docker Example + _See [Docker instructions](https://maplibre.org/martin/run-with-docker.html) in the Martin book._ Martin is also available as a [Docker image](https://ghcr.io/maplibre/martin). You could either share a configuration file from the host with the container via the `-v` param, or you can let Martin auto-discover all sources e.g. by passing `DATABASE_URL` or specifying the .mbtiles/.pmtiles files. @@ -86,6 +89,7 @@ docker run -p 3000:3000 \ ``` ## API + _See [API documentation](https://maplibre.org/martin/using.html) in the Martin book._ Martin data is available via the HTTP `GET` endpoints: @@ -104,6 +108,7 @@ Martin data is available via the HTTP `GET` endpoints: | `/health` | Martin server health check: returns 200 `OK` | ## Documentation + See [Martin book](https://maplibre.org/martin/) for complete documentation. ## License diff --git a/docs/src/using.md b/docs/src/using.md index 30650848b..2d854b23f 100644 --- a/docs/src/using.md +++ b/docs/src/using.md @@ -16,9 +16,11 @@ Martin data is available via the HTTP `GET` endpoints: | `/health` | Martin server health check: returns 200 `OK` | ### Duplicate Source ID + In case there is more than one source that has the same name, e.g. a PG function is available in two schemas/connections, or a table has more than one geometry columns, sources will be assigned unique IDs such as `/points`, `/points.1`, etc. ### Reserved Source IDs + Some source IDs are reserved for internal use. If you try to use them, they will be automatically renamed to a unique ID the same way as duplicate source IDs are handled, e.g. a `catalog` source will become `catalog.1`. Some of the reserved IDs: `_`, `catalog`, `config`, `font`, `health`, `help`, `index`, `manifest`, `metrics`, `refresh`, From 445cc2126125cec7789b2a9799c4f0a2ac047980 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 20:50:32 +0800 Subject: [PATCH 08/37] Eliminate `MD022/blanks-around-headings` warnings --- docs/src/sources-sprites.md | 4 ++++ docs/src/tools.md | 2 ++ 2 files changed, 6 insertions(+) diff --git a/docs/src/sources-sprites.md b/docs/src/sources-sprites.md index a5d0a91b8..809e87092 100644 --- a/docs/src/sources-sprites.md +++ b/docs/src/sources-sprites.md @@ -3,6 +3,7 @@ Given a directory with SVG images, Martin will generate a sprite -- a JSON index and a PNG image, for both low and high resolution displays. The SVG filenames without extension will be used as the sprite image IDs. The images are searched recursively in the given directory, so subdirectory names will be used as prefixes for the image IDs, e.g. `icons/bicycle.svg` will be available as `icons/bicycle` sprite image. The sprite generation is not yet cached, and may require external reverse proxy or CDN for faster operation. ### API + Martin uses [MapLibre sprites API](https://maplibre.org/maplibre-style-spec/sprite/) specification to serve sprites via several endpoints. The sprite image and index are generated on the fly, so if the sprite directory is updated, the changes will be reflected immediately. ##### Sprite PNG @@ -12,6 +13,7 @@ Martin uses [MapLibre sprites API](https://maplibre.org/maplibre-style-spec/spri `GET /sprite/.png` endpoint contains a single PNG sprite image that combines all sources images. Additionally, there is a high DPI version available at `GET /sprite/@2x.png`. ##### Sprite index + `/sprite/.json` metadata index describing the position and size of each image inside the sprite. Just like the PNG, there is a high DPI version available at `/sprite/@2x.json`. ```json @@ -26,7 +28,9 @@ Martin uses [MapLibre sprites API](https://maplibre.org/maplibre-style-spec/spri ... } ``` + #### Combining Multiple Sprites + Multiple sprite_id values can be combined into one sprite with the same pattern as for tile joining: `/sprite/,,...,`. No ID renaming is done, so identical sprite names will override one another. ### Configuring from CLI diff --git a/docs/src/tools.md b/docs/src/tools.md index 08a7b135a..a7b13e662 100644 --- a/docs/src/tools.md +++ b/docs/src/tools.md @@ -3,9 +3,11 @@ Martin project contains additional tooling to help manage the data servable with Martin tile server. ## `martin-cp` + `martin-cp` is a tool for generating tiles in bulk, and save retrieved tiles into a new or an existing MBTiles file. It can be used to generate tiles for a large area or multiple areas. If multiple areas overlap, it will generate tiles only once. `martin-cp` supports the same configuration file and CLI arguments as Martin server, so it can support all sources and even combining sources. ## `mbtiles` + `mbtiles` is a small utility to interact with the `*.mbtiles` files from the command line. It allows users to examine, copy, validate, compare, and apply diffs between them. Use `mbtiles --help` to see a list of available commands, and `mbtiles --help` to see help for a specific command. From d75a1624fba670089e7851083cc9ee669d911153 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Wed, 29 Nov 2023 21:09:47 +0800 Subject: [PATCH 09/37] Run `markdownlint-cli2 --fix` --- README.md | 1 - docs/src/martin-cp.md | 2 +- docs/src/recipes.md | 1 + docs/src/run-with-docker.md | 1 + docs/src/sources-fonts.md | 3 ++- docs/src/sources-pg-functions.md | 2 ++ docs/src/sources-pg-tables.md | 2 ++ docs/src/using-with-maplibre.md | 1 - 8 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 11346400b..d65ebf78c 100755 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ _See [installation instructions](https://maplibre.org/martin/installation.html) **Prerequisites:** If using Martin with PostgreSQL database, you must install PostGIS with at least v3.0+, v3.1+ recommended. - You can download martin from [GitHub releases page](https://github.com/maplibre/martin/releases). | Platform | AMD-64 | ARM-64 | diff --git a/docs/src/martin-cp.md b/docs/src/martin-cp.md index 3d3402026..b8533f433 100644 --- a/docs/src/martin-cp.md +++ b/docs/src/martin-cp.md @@ -4,7 +4,7 @@ ## Usage -This copies tiles from a PostGIS table `my_table` into an MBTiles file `tileset.mbtiles` using [normalized](mbtiles-schema.md) schema, with zoom levels from 0 to 10, and bounds of the whole world. +This copies tiles from a PostGIS table `my_table` into an MBTiles file `tileset.mbtiles` using [normalized](mbtiles-schema.md) schema, with zoom levels from 0 to 10, and bounds of the whole world. ```shell martin-cp --output-file tileset.mbtiles \ diff --git a/docs/src/recipes.md b/docs/src/recipes.md index fb2dfe6d9..2c864efeb 100644 --- a/docs/src/recipes.md +++ b/docs/src/recipes.md @@ -31,6 +31,7 @@ martin ``` You may also be able to validate SSL certificate with an explicit sslmode, e.g. + ```shell export DATABASE_URL="$(heroku config:get DATABASE_URL -a APP_NAME)?sslmode=verify-ca" ``` diff --git a/docs/src/run-with-docker.md b/docs/src/run-with-docker.md index 5b60bf445..3f4714f5d 100644 --- a/docs/src/run-with-docker.md +++ b/docs/src/run-with-docker.md @@ -3,6 +3,7 @@ You can use official Docker image [`ghcr.io/maplibre/martin`](https://ghcr.io/maplibre/martin) ### Using Non-Local PostgreSQL + ```shell docker run \ -p 3000:3000 \ diff --git a/docs/src/sources-fonts.md b/docs/src/sources-fonts.md index 09199eb77..21095ca8e 100644 --- a/docs/src/sources-fonts.md +++ b/docs/src/sources-fonts.md @@ -4,6 +4,7 @@ Martin can serve glyph ranges from `otf`, `ttf`, and `ttc` fonts as needed by Ma The glyph range generation is not yet cached, and may require external reverse proxy or CDN for faster operation. ## API + Fonts ranges are available either for a single font, or a combination of multiple fonts. The font names are case-sensitive and should match the font name in the font file as published in the catalog. Make sure to URL-escape font names as they usually contain spaces. | | Font Request | @@ -11,7 +12,6 @@ Fonts ranges are available either for a single font, or a combination of multipl | Pattern | `/font/{name}/{start}-{end}` | | Example | `/font/Overpass%20Mono%20Bold/0-255` | - ### Composite Font Request When combining multiple fonts, the glyph range will contain glyphs from the first listed font if available, and fallback to the next font if the glyph is not available in the first font, etc. The glyph range will be empty if none of the fonts contain the glyph. @@ -22,6 +22,7 @@ When combining multiple fonts, the glyph range will contain glyphs from the firs | Example | `/font/Overpass%20Mono%20Bold,Overpass%20Mono%20Light/0-255` | ### Catalog + Martin will show all available fonts at the `/catalog` endpoint. ```shell diff --git a/docs/src/sources-pg-functions.md b/docs/src/sources-pg-functions.md index b18c83862..b00accdce 100644 --- a/docs/src/sources-pg-functions.md +++ b/docs/src/sources-pg-functions.md @@ -10,6 +10,7 @@ Function Source is a database function which can be used to query [vector tiles] | query (optional, any name) | json | Query string parameters | ### Simple Function + For example, if you have a table `table_source` in WGS84 (`4326` SRID), then you can use this function as a Function Source: ```sql, ignore @@ -35,6 +36,7 @@ $$ LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE; ``` ### Function with Query Parameters + Users may add a `query` parameter to pass additional parameters to the function. _**TODO**: Modify this example to actually use the query parameters._ diff --git a/docs/src/sources-pg-tables.md b/docs/src/sources-pg-tables.md index 6ca261c52..51968d5cc 100644 --- a/docs/src/sources-pg-tables.md +++ b/docs/src/sources-pg-tables.md @@ -9,11 +9,13 @@ For example, if there is a table `public.table_source`: the default `TileJSON` might look like this (note that URL will be automatically adjusted to match the request host): The table: + ```sql CREATE TABLE "public"."table_source" ( "gid" int4 NOT NULL, "geom" "public"."geometry" ); ``` The TileJSON: + ```json { "tilejson": "3.0.0", diff --git a/docs/src/using-with-maplibre.md b/docs/src/using-with-maplibre.md index cb319be52..a007ddf40 100644 --- a/docs/src/using-with-maplibre.md +++ b/docs/src/using-with-maplibre.md @@ -4,7 +4,6 @@ You can add a layer to the map and specify Martin [TileJSON](https://github.com/mapbox/tilejson-spec) endpoint as a vector source URL. You should also specify a `source-layer` property. For [Table Sources](sources-pg-tables.md) it is `{table_name}` by default. - ```js map.addLayer({ id: 'points', From 688c9d6212c69ae37b700b6bb7733398f5b9368d Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Thu, 30 Nov 2023 10:48:03 +0800 Subject: [PATCH 10/37] Ignore MD013 rule --- .github/files/.markdownlint-cli2.jsonc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/files/.markdownlint-cli2.jsonc b/.github/files/.markdownlint-cli2.jsonc index f5eea1eec..cede4a360 100644 --- a/.github/files/.markdownlint-cli2.jsonc +++ b/.github/files/.markdownlint-cli2.jsonc @@ -1,7 +1,9 @@ { "config": { "default": true, - "relative-links": true + "relative-links": true, + // Line length Check. See https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md + "MD013": false }, // globs expression. See https://github.com/DavidAnson/markdownlint-cli2#command-line "globs": [ From db70bf137b641908f2c861c24137e441e29a4614 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 09:12:35 +0800 Subject: [PATCH 11/37] Ignore MD041, MD033 --- .github/files/.markdownlint-cli2.jsonc | 4 +++- docs/src/SUMMARY.md | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/files/.markdownlint-cli2.jsonc b/.github/files/.markdownlint-cli2.jsonc index cede4a360..c49e79560 100644 --- a/.github/files/.markdownlint-cli2.jsonc +++ b/.github/files/.markdownlint-cli2.jsonc @@ -3,7 +3,9 @@ "default": true, "relative-links": true, // Line length Check. See https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md - "MD013": false + "MD013": false, + "MD041": false, + "MD033": false }, // globs expression. See https://github.com/DavidAnson/markdownlint-cli2#command-line "globs": [ diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 720ab47ef..3bdb947d3 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -1,4 +1,5 @@ [Introduction](introduction.md) + - [Installation](installation.md) - [Running](run.md) - [Command Line Interface](run-with-cli.md) From 316169c99fca3b51e4d13fac098e1947eb03bf49 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 09:21:28 +0800 Subject: [PATCH 12/37] Add relative link check --- .github/files/.markdownlint-cli2.jsonc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/files/.markdownlint-cli2.jsonc b/.github/files/.markdownlint-cli2.jsonc index c49e79560..e2658d60c 100644 --- a/.github/files/.markdownlint-cli2.jsonc +++ b/.github/files/.markdownlint-cli2.jsonc @@ -12,6 +12,6 @@ "README.md", "!CHANGELOG.md", "docs/src/*.md" - ] - // "customRules": ["markdownlint-rule-relative-links"] + ], + "customRules": ["markdownlint-rule-relative-links"] } From aca15c353cabb4708a2265232dbf0101b4d06bbf Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 09:58:57 +0800 Subject: [PATCH 13/37] Use markdown-link-check github action --- .github/files/.markdownlint-cli2.jsonc | 3 +-- .github/workflows/build-deploy-docs.yml | 6 ++++++ docs/src/config-file.md | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/files/.markdownlint-cli2.jsonc b/.github/files/.markdownlint-cli2.jsonc index e2658d60c..0d8c75194 100644 --- a/.github/files/.markdownlint-cli2.jsonc +++ b/.github/files/.markdownlint-cli2.jsonc @@ -12,6 +12,5 @@ "README.md", "!CHANGELOG.md", "docs/src/*.md" - ], - "customRules": ["markdownlint-rule-relative-links"] + ] } diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index 7f8f725e1..006bb768c 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -15,6 +15,12 @@ jobs: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 + - name: Run link check + uses: gaurav-nelson/github-action-markdown-link-check@v1 + with: + use-quiet-mode: 'no' + use-verbose-mode: 'yes' + - name: Markdown lint uses: DavidAnson/markdownlint-cli2-action@v14 with: diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 5e7257abc..95fc5a337 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -1,6 +1,6 @@ # Configuration File -If you don't want to expose all of your tables and functions, you can list your sources in a configuration file. To start Martin with a configuration file you need to pass a path to a file with a `--config` argument. Config files may contain environment variables, which will be expanded before parsing. For example, to use `MY_DATABASE_URL` in your config file: `connection_string: ${MY_DATABASE_URL}`, or with a default `connection_string: ${MY_DATABASE_URL:-postgresql://postgres@localhost/db}` +If[not_exist_will_remove_before_merge_just_for_test.md] you don't want to expose all of your tables and functions, you can list your sources in a configuration file. To start Martin with a configuration file you need to pass a path to a file with a `--config` argument. Config files may contain environment variables, which will be expanded before parsing. For example, to use `MY_DATABASE_URL` in your config file: `connection_string: ${MY_DATABASE_URL}`, or with a default `connection_string: ${MY_DATABASE_URL:-postgresql://postgres@localhost/db}` ```shell martin --config config.yaml From 407e642665d61bd43571790834bf4a395f77029d Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 10:27:07 +0800 Subject: [PATCH 14/37] Set link check path --- .github/workflows/build-deploy-docs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index 006bb768c..425b1bb02 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -20,6 +20,8 @@ jobs: with: use-quiet-mode: 'no' use-verbose-mode: 'yes' + folder-path: 'docs/src' + file-path: './README.md' - name: Markdown lint uses: DavidAnson/markdownlint-cli2-action@v14 From 58b335aca1426367b7417ad085ccdec6817af12a Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 10:59:44 +0800 Subject: [PATCH 15/37] Ignore localhost --- .github/files/markdown.links.config.json | 10 ++++++++++ .github/workflows/build-deploy-docs.yml | 1 + 2 files changed, 11 insertions(+) create mode 100644 .github/files/markdown.links.config.json diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json new file mode 100644 index 000000000..dbb9bc18b --- /dev/null +++ b/.github/files/markdown.links.config.json @@ -0,0 +1,10 @@ +{ + "ignorePatterns": [ + { + "pattern": "^http://127.0.0.1" + }, + { + "pattern": "^http://localhost" + } + ] +} \ No newline at end of file diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index 425b1bb02..fd7bbad3d 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -22,6 +22,7 @@ jobs: use-verbose-mode: 'yes' folder-path: 'docs/src' file-path: './README.md' + config-file: './.github/files/markdown.links.config.json' - name: Markdown lint uses: DavidAnson/markdownlint-cli2-action@v14 From a728eaf3a8f98baaf0aefa105373d2bfbc23cbc1 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 11:09:10 +0800 Subject: [PATCH 16/37] fix link in run-with-nginx.md --- docs/src/run-with-nginx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/run-with-nginx.md b/docs/src/run-with-nginx.md index 235746922..544588fc0 100644 --- a/docs/src/run-with-nginx.md +++ b/docs/src/run-with-nginx.md @@ -91,4 +91,4 @@ http { } ``` -You can find an example NGINX configuration file [here](https://github.com/maplibre/martin/blob/main/nginx.conf). +You can find an example NGINX configuration file [here](https://github.com/maplibre/martin/blob/main/demo/frontend/nginx.conf). From 20a33d81da792beefd3aed96d9aec315772c8cb1 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 11:38:30 +0800 Subject: [PATCH 17/37] Add httpheader and timeout --- .github/files/markdown.links.config.json | 12 +++++++++++- docs/src/config-file.md | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index dbb9bc18b..b1b797216 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -1,4 +1,5 @@ { + "timeout": "3000ms", "ignorePatterns": [ { "pattern": "^http://127.0.0.1" @@ -6,5 +7,14 @@ { "pattern": "^http://localhost" } - ] + ], + "httpHeaders": [ + { + "urls": ["https://crates.io", "http://opensource.org"], + "headers": { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", + "Accept": "/" + } + } + ] } \ No newline at end of file diff --git a/docs/src/config-file.md b/docs/src/config-file.md index 95fc5a337..5e7257abc 100644 --- a/docs/src/config-file.md +++ b/docs/src/config-file.md @@ -1,6 +1,6 @@ # Configuration File -If[not_exist_will_remove_before_merge_just_for_test.md] you don't want to expose all of your tables and functions, you can list your sources in a configuration file. To start Martin with a configuration file you need to pass a path to a file with a `--config` argument. Config files may contain environment variables, which will be expanded before parsing. For example, to use `MY_DATABASE_URL` in your config file: `connection_string: ${MY_DATABASE_URL}`, or with a default `connection_string: ${MY_DATABASE_URL:-postgresql://postgres@localhost/db}` +If you don't want to expose all of your tables and functions, you can list your sources in a configuration file. To start Martin with a configuration file you need to pass a path to a file with a `--config` argument. Config files may contain environment variables, which will be expanded before parsing. For example, to use `MY_DATABASE_URL` in your config file: `connection_string: ${MY_DATABASE_URL}`, or with a default `connection_string: ${MY_DATABASE_URL:-postgresql://postgres@localhost/db}` ```shell martin --config config.yaml From 072c5cb06901147891fd4767017b8be251393206 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 13:21:34 +0800 Subject: [PATCH 18/37] Update link check config --- .github/files/markdown.links.config.json | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index b1b797216..cae29ee81 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -1,20 +1,24 @@ { - "timeout": "3000ms", - "ignorePatterns": [ + "ignorePatterns": [ { "pattern": "^http://127.0.0.1" }, { "pattern": "^http://localhost" + }, + { + "pattern": "^https://img.shields.io/badge" } - ], + ], "httpHeaders": [ { - "urls": ["https://crates.io", "http://opensource.org"], - "headers": { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", - "Accept": "/" - } + "urls": [ + "https://crates.io", "https://github.com" + ], + "headers": { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7" + } } - ] + ] } \ No newline at end of file From e13654112f0cc173c8fbd546f4b2029deb00cbe2 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 14:29:13 +0800 Subject: [PATCH 19/37] Add headers for opensource.org --- .github/files/markdown.links.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index cae29ee81..f74c7d009 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -13,7 +13,7 @@ "httpHeaders": [ { "urls": [ - "https://crates.io", "https://github.com" + "https://crates.io", "https://github.com", "http://opensource.org" ], "headers": { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", From 63f7188fbfb08884c9cc1b504eeb6793d8502b4e Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 14:41:37 +0800 Subject: [PATCH 20/37] Update license link --- .github/files/markdown.links.config.json | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index f74c7d009..19f1f8f0d 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -13,7 +13,7 @@ "httpHeaders": [ { "urls": [ - "https://crates.io", "https://github.com", "http://opensource.org" + "https://crates.io", "https://github.com", "https://opensource.org" ], "headers": { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", diff --git a/README.md b/README.md index d65ebf78c..a97ff8a3d 100755 --- a/README.md +++ b/README.md @@ -114,8 +114,8 @@ See [Martin book](https://maplibre.org/martin/) for complete documentation. Licensed under either of -* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) -* MIT license ([LICENSE-MIT](LICENSE-MIT) or ) +* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) +* MIT license ([LICENSE-MIT](LICENSE-MIT) or ) at your option. ### Contribution From 87da4c7f4bd2f5dd2edc43049af675ebc1880a2f Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 14:52:17 +0800 Subject: [PATCH 21/37] Update headers for opensource.org --- .github/files/markdown.links.config.json | 11 ++++++++--- README.md | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index 19f1f8f0d..ca4d87998 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -13,12 +13,17 @@ "httpHeaders": [ { "urls": [ - "https://crates.io", "https://github.com", "https://opensource.org" + "https://crates.io", + "https://github.com", + "https://opensource.org", + "http://opensource.org" ], "headers": { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7" + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6" } } ] -} \ No newline at end of file +} diff --git a/README.md b/README.md index a97ff8a3d..d65ebf78c 100755 --- a/README.md +++ b/README.md @@ -114,8 +114,8 @@ See [Martin book](https://maplibre.org/martin/) for complete documentation. Licensed under either of -* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) -* MIT license ([LICENSE-MIT](LICENSE-MIT) or ) +* Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or ) +* MIT license ([LICENSE-MIT](LICENSE-MIT) or ) at your option. ### Contribution From 88b454871ffd6fa9b5a164dd043b5c57506e837e Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 16:00:57 +0800 Subject: [PATCH 22/37] Rename markdown lint config file --- ...{.markdownlint-cli2.jsonc => martin.markdownlint-cli2.jsonc} | 0 .github/workflows/build-deploy-docs.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/files/{.markdownlint-cli2.jsonc => martin.markdownlint-cli2.jsonc} (100%) diff --git a/.github/files/.markdownlint-cli2.jsonc b/.github/files/martin.markdownlint-cli2.jsonc similarity index 100% rename from .github/files/.markdownlint-cli2.jsonc rename to .github/files/martin.markdownlint-cli2.jsonc diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index fd7bbad3d..0a7fe88ff 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -27,7 +27,7 @@ jobs: - name: Markdown lint uses: DavidAnson/markdownlint-cli2-action@v14 with: - config: './.github/files/.markdownlint-cli2.jsonc' + config: './.github/files/martin.markdownlint-cli2.jsonc' - name: Setup mdBook uses: peaceiris/actions-mdbook@v1 From 02b3bf75d6bcc1dede50a3e5056bc17ec83db16e Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 16:44:58 +0800 Subject: [PATCH 23/37] Add markdown fmt to jsut fmt --- justfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 19173505c..b8796ff08 100644 --- a/justfile +++ b/justfile @@ -250,8 +250,10 @@ print-conn-str: lint: fmt clippy # Run cargo fmt -fmt: +fmt: (install-markdownlint-cli2) cargo fmt --all -- --check + @echo "Running markdownlint-cli2..." + markdownlint-cli2 --config ".github/files/martin.markdownlint-cli2.jsonc" # Run Nightly cargo fmt, ordering imports fmt2: @@ -290,3 +292,11 @@ cargo-install $COMMAND $INSTALL_CMD="" *ARGS="": echo "$COMMAND could not be found. Installing it with cargo install ${INSTALL_CMD:-$COMMAND} {{ ARGS }}" ;\ cargo install ${INSTALL_CMD:-$COMMAND} {{ ARGS }} ;\ fi + +# Check if markdownlint-cli2 is installed, and install it if needed +[private] +install-markdownlint-cli2: + @if ! command -v markdownlint-cli2 &> /dev/null; then \ + echo "markdownlint-cli2 could not be found. Installing it with npm install --global markdownlint-cli2" ;\ + sudo npm install --global markdownlint-cli2 ;\ + fi \ No newline at end of file From eeae4b09e8aca1ef27ded8c262903d581bc22fd9 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Fri, 1 Dec 2023 17:03:14 +0800 Subject: [PATCH 24/37] Add link check to just clippy --- justfile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index b8796ff08..f65fd04cd 100644 --- a/justfile +++ b/justfile @@ -260,9 +260,10 @@ fmt2: cargo +nightly fmt -- --config imports_granularity=Module,group_imports=StdExternalCrate # Run cargo clippy -clippy: +clippy: (install-markdown-link-check) cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace + find . -name \*.md -print0 | xargs -0 -n1 markdown-link-check --config ".github/files/markdown.links.config.json" # These steps automatically run before git push via a git hook [private] @@ -299,4 +300,12 @@ install-markdownlint-cli2: @if ! command -v markdownlint-cli2 &> /dev/null; then \ echo "markdownlint-cli2 could not be found. Installing it with npm install --global markdownlint-cli2" ;\ sudo npm install --global markdownlint-cli2 ;\ + fi + +# Check if markdown-link-check is installed, and install it if needed +[private] +install-markdown-link-check: + @if ! command -v markdown-link-check &> /dev/null; then \ + echo "markdown-link-check could not be found. Installing it with npm install -g markdown-link-check" ;\ + sudo npm install -g markdown-link-check ;\ fi \ No newline at end of file From 911aa3875f915fc1e8abf4aba8780503b7d3aeed Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 2 Dec 2023 10:09:18 +0800 Subject: [PATCH 25/37] Remove img.shields.io from ignore list --- .github/files/markdown.links.config.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index ca4d87998..2cb11dc2e 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -5,9 +5,6 @@ }, { "pattern": "^http://localhost" - }, - { - "pattern": "^https://img.shields.io/badge" } ], "httpHeaders": [ From b0ec469f855b80ba18a8a45b5df0c47044ddfd3c Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 2 Dec 2023 11:24:48 +0800 Subject: [PATCH 26/37] Add --fix to just fmt --- justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/justfile b/justfile index f65fd04cd..65fe4037e 100644 --- a/justfile +++ b/justfile @@ -253,7 +253,7 @@ lint: fmt clippy fmt: (install-markdownlint-cli2) cargo fmt --all -- --check @echo "Running markdownlint-cli2..." - markdownlint-cli2 --config ".github/files/martin.markdownlint-cli2.jsonc" + markdownlint-cli2 --config ".github/files/martin.markdownlint-cli2.jsonc" --fix # Run Nightly cargo fmt, ordering imports fmt2: From d0d0da5fba74fd06b43ddb5cd6cf2fd9fcdddb51 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 2 Dec 2023 11:26:23 +0800 Subject: [PATCH 27/37] Ignore opensource.org --- .github/files/markdown.links.config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index 2cb11dc2e..734b7fd6c 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -5,6 +5,9 @@ }, { "pattern": "^http://localhost" + }, + { + "pattern": "^http://opensource.org" } ], "httpHeaders": [ From 74d9b63663c8a17352fe09eb57dd079f48feef90 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 2 Dec 2023 11:34:16 +0800 Subject: [PATCH 28/37] Ignore MD045 MD001 in markdown lintting --- .github/files/martin.markdownlint-cli2.jsonc | 30 +++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/files/martin.markdownlint-cli2.jsonc b/.github/files/martin.markdownlint-cli2.jsonc index 0d8c75194..f1f24a814 100644 --- a/.github/files/martin.markdownlint-cli2.jsonc +++ b/.github/files/martin.markdownlint-cli2.jsonc @@ -1,16 +1,18 @@ { - "config": { - "default": true, - "relative-links": true, - // Line length Check. See https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md - "MD013": false, - "MD041": false, - "MD033": false - }, - // globs expression. See https://github.com/DavidAnson/markdownlint-cli2#command-line - "globs": [ - "README.md", - "!CHANGELOG.md", - "docs/src/*.md" - ] + "config": { + "default": true, + "relative-links": true, + // Line length Check. See https://github.com/DavidAnson/markdownlint/blob/v0.32.1/doc/md013.md + "MD013": false, + "MD041": false, + "MD033": false, + "MD045": false, + "MD001": false + }, + // globs expression. See https://github.com/DavidAnson/markdownlint-cli2#command-line + "globs": [ + "README.md", + "!CHANGELOG.md", + "docs/src/*.md" + ] } From 186155e38298e0374dbdd4bc8edd85d73f9be501 Mon Sep 17 00:00:00 2001 From: sharkAndshark Date: Sat, 2 Dec 2023 11:39:53 +0800 Subject: [PATCH 29/37] Update martin-cp.md --- docs/src/martin-cp.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/src/martin-cp.md b/docs/src/martin-cp.md index bcb80c386..53e06b319 100644 --- a/docs/src/martin-cp.md +++ b/docs/src/martin-cp.md @@ -17,5 +17,3 @@ martin-cp --output-file tileset.mbtiles \ --source source_name \ postgresql://postgres@localhost:5432/db ``` - -You From c0f998712cc842cb22344607962c3013e7f931bb Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 4 Dec 2023 01:21:14 +0000 Subject: [PATCH 30/37] Remove opensource.org from httpheaders --- .github/files/markdown.links.config.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index 734b7fd6c..a8a706b09 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -14,9 +14,7 @@ { "urls": [ "https://crates.io", - "https://github.com", - "https://opensource.org", - "http://opensource.org" + "https://github.com" ], "headers": { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", From aae73ea54fddffe5b44081b10811b3e43c72ada7 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 4 Dec 2023 01:22:14 +0000 Subject: [PATCH 31/37] Remove NPM packages --- justfile | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/justfile b/justfile index fce052c49..031a0b831 100644 --- a/justfile +++ b/justfile @@ -296,19 +296,3 @@ cargo-install $COMMAND $INSTALL_CMD="" *ARGS="": echo "$COMMAND could not be found. Installing it with cargo install ${INSTALL_CMD:-$COMMAND} {{ ARGS }}" ;\ cargo install ${INSTALL_CMD:-$COMMAND} {{ ARGS }} ;\ fi - -# Check if markdownlint-cli2 is installed, and install it if needed -[private] -install-markdownlint-cli2: - @if ! command -v markdownlint-cli2 &> /dev/null; then \ - echo "markdownlint-cli2 could not be found. Installing it with npm install --global markdownlint-cli2" ;\ - sudo npm install --global markdownlint-cli2 ;\ - fi - -# Check if markdown-link-check is installed, and install it if needed -[private] -install-markdown-link-check: - @if ! command -v markdown-link-check &> /dev/null; then \ - echo "markdown-link-check could not be found. Installing it with npm install -g markdown-link-check" ;\ - sudo npm install -g markdown-link-check ;\ - fi \ No newline at end of file From 307f60d6b2a23da87ba484138cf21cd210fd537c Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 4 Dec 2023 01:30:25 +0000 Subject: [PATCH 32/37] Add fmt-md to justfile --- justfile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/justfile b/justfile index 031a0b831..e164dd23f 100644 --- a/justfile +++ b/justfile @@ -253,17 +253,21 @@ print-conn-str: lint: fmt clippy # Run cargo fmt -fmt: (install-markdownlint-cli2) +fmt: cargo fmt --all -- --check - @echo "Running markdownlint-cli2..." markdownlint-cli2 --config ".github/files/martin.markdownlint-cli2.jsonc" --fix +# Run markdown lintting +fmt-md: + @echo "Running markdownlint-cli2..." + docker run -it --rm -v $PWD:/workdir davidanson/markdownlint-cli2 --config /workdir/.github/files/martin.markdownlint-cli2.jsonc --fix + # Run Nightly cargo fmt, ordering imports fmt2: cargo +nightly fmt -- --config imports_granularity=Module,group_imports=StdExternalCrate # Run cargo clippy -clippy: (install-markdown-link-check) +clippy: cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace find . -name \*.md -print0 | xargs -0 -n1 markdown-link-check --config ".github/files/markdown.links.config.json" From f70049209c3e7dbbc77e5dc51642874b030f0c98 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 4 Dec 2023 02:09:35 +0000 Subject: [PATCH 33/37] Move link check to just clippy-md --- justfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/justfile b/justfile index e164dd23f..b0e17dbe5 100644 --- a/justfile +++ b/justfile @@ -272,6 +272,12 @@ clippy: RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace find . -name \*.md -print0 | xargs -0 -n1 markdown-link-check --config ".github/files/markdown.links.config.json" +# Run markdown links checking +clippy-md: + @echo "Running markdown-link-check..." + find ./docs/src -name \*.md -print0 | xargs -0 -n1 -I{} docker run -i --rm -v ${PWD}:/workdir ghcr.io/tcort/markdown-link-check --config /workdir/.github/files/markdown.links.config.json /workdir/{} + docker run -i --rm -v ${PWD}:/workdir ghcr.io/tcort/markdown-link-check --config /workdir/.github/files/markdown.links.config.json /workdir/README.md + # These steps automatically run before git push via a git hook [private] git-pre-push: env-info restart lint test From cd0847aed38dc60d9a5235fd488b8035b9029a86 Mon Sep 17 00:00:00 2001 From: Lucas Date: Mon, 4 Dec 2023 02:16:50 +0000 Subject: [PATCH 34/37] Update link checking headers --- .github/files/markdown.links.config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index a8a706b09..77801c7f4 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -20,7 +20,7 @@ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", "Accept-Encoding": "gzip, deflate, br", - "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6" + "Accept-Language": "en" } } ] From fdac18ee32719b5a85678bbcda4aa9647acbf0af Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 4 Dec 2023 16:49:54 -0500 Subject: [PATCH 35/37] optimize link check --- ...2.jsonc => config.markdownlint-cli2.jsonc} | 0 .github/files/markdown.links.config.json | 59 +++++++++++-------- .github/workflows/build-deploy-docs.yml | 4 +- justfile | 10 ++-- 4 files changed, 40 insertions(+), 33 deletions(-) rename .github/files/{martin.markdownlint-cli2.jsonc => config.markdownlint-cli2.jsonc} (100%) diff --git a/.github/files/martin.markdownlint-cli2.jsonc b/.github/files/config.markdownlint-cli2.jsonc similarity index 100% rename from .github/files/martin.markdownlint-cli2.jsonc rename to .github/files/config.markdownlint-cli2.jsonc diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index 77801c7f4..008617654 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -1,27 +1,36 @@ { - "ignorePatterns": [ - { - "pattern": "^http://127.0.0.1" - }, - { - "pattern": "^http://localhost" - }, - { - "pattern": "^http://opensource.org" - } - ], - "httpHeaders": [ - { - "urls": [ - "https://crates.io", - "https://github.com" - ], - "headers": { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", - "Accept-Encoding": "gzip, deflate, br", - "Accept-Language": "en" - } - } - ] + "replacementPatterns": [ + { + "__comment__": "See https://github.com/tcort/markdown-link-check/issues/264", + "pattern": "%23", + "replacement": "" + } + ], + "timeout": "2s", + "retryOn429": true, + "ignorePatterns": [ + { + "pattern": "^http://127.0.0.1" + }, + { + "pattern": "^http://localhost" + }, + { + "pattern": "^http://opensource.org" + } + ], + "httpHeaders": [ + { + "urls": [ + "https://crates.io", + "https://github.com" + ], + "headers": { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "en" + } + } + ] } diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index 0a7fe88ff..759636909 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -22,12 +22,12 @@ jobs: use-verbose-mode: 'yes' folder-path: 'docs/src' file-path: './README.md' - config-file: './.github/files/markdown.links.config.json' + config-file: '.github/files/markdown.links.config.json' - name: Markdown lint uses: DavidAnson/markdownlint-cli2-action@v14 with: - config: './.github/files/martin.markdownlint-cli2.jsonc' + config: '.github/files/config.markdownlint-cli2.jsonc' - name: Setup mdBook uses: peaceiris/actions-mdbook@v1 diff --git a/justfile b/justfile index b0e17dbe5..daabc9111 100644 --- a/justfile +++ b/justfile @@ -255,12 +255,12 @@ lint: fmt clippy # Run cargo fmt fmt: cargo fmt --all -- --check - markdownlint-cli2 --config ".github/files/martin.markdownlint-cli2.jsonc" --fix + markdownlint-cli2 --config ".github/files/config.markdownlint-cli2.jsonc" --fix # Run markdown lintting fmt-md: @echo "Running markdownlint-cli2..." - docker run -it --rm -v $PWD:/workdir davidanson/markdownlint-cli2 --config /workdir/.github/files/martin.markdownlint-cli2.jsonc --fix + docker run -it --rm -v $PWD:/workdir davidanson/markdownlint-cli2 --config /workdir/.github/files/config.markdownlint-cli2.jsonc --fix # Run Nightly cargo fmt, ordering imports fmt2: @@ -270,13 +270,11 @@ fmt2: clippy: cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace - find . -name \*.md -print0 | xargs -0 -n1 markdown-link-check --config ".github/files/markdown.links.config.json" # Run markdown links checking clippy-md: - @echo "Running markdown-link-check..." - find ./docs/src -name \*.md -print0 | xargs -0 -n1 -I{} docker run -i --rm -v ${PWD}:/workdir ghcr.io/tcort/markdown-link-check --config /workdir/.github/files/markdown.links.config.json /workdir/{} - docker run -i --rm -v ${PWD}:/workdir ghcr.io/tcort/markdown-link-check --config /workdir/.github/files/markdown.links.config.json /workdir/README.md + @echo "Running markdown-link-check to validate " + docker run -it --rm -v ${PWD}:/workdir --entrypoint sh ghcr.io/tcort/markdown-link-check -c 'echo -e "/workdir/README.md\n$(find /workdir/docs/src -name "*.md")"|tr "\n" "\0" | xargs -0 -P 5 -n1 -I{} markdown-link-check --config /workdir/.github/files/markdown.links.config.json {}' # These steps automatically run before git push via a git hook [private] From 8c032499fffcded643d6152e7a61a195e6d7cd0c Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 4 Dec 2023 16:55:09 -0500 Subject: [PATCH 36/37] minor doc --- .github/workflows/build-deploy-docs.yml | 12 ++++++------ justfile | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-deploy-docs.yml b/.github/workflows/build-deploy-docs.yml index 759636909..c3aaba825 100644 --- a/.github/workflows/build-deploy-docs.yml +++ b/.github/workflows/build-deploy-docs.yml @@ -15,7 +15,12 @@ jobs: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 - - name: Run link check + - name: 'Validate .md files (use "just fmt-md" to fix)' + uses: DavidAnson/markdownlint-cli2-action@v14 + with: + config: '.github/files/config.markdownlint-cli2.jsonc' + + - name: 'Check Markdown URLs (same as "just clippy-md")' uses: gaurav-nelson/github-action-markdown-link-check@v1 with: use-quiet-mode: 'no' @@ -24,11 +29,6 @@ jobs: file-path: './README.md' config-file: '.github/files/markdown.links.config.json' - - name: Markdown lint - uses: DavidAnson/markdownlint-cli2-action@v14 - with: - config: '.github/files/config.markdownlint-cli2.jsonc' - - name: Setup mdBook uses: peaceiris/actions-mdbook@v1 with: diff --git a/justfile b/justfile index daabc9111..3b7a80c4e 100644 --- a/justfile +++ b/justfile @@ -273,8 +273,8 @@ clippy: # Run markdown links checking clippy-md: - @echo "Running markdown-link-check to validate " - docker run -it --rm -v ${PWD}:/workdir --entrypoint sh ghcr.io/tcort/markdown-link-check -c 'echo -e "/workdir/README.md\n$(find /workdir/docs/src -name "*.md")"|tr "\n" "\0" | xargs -0 -P 5 -n1 -I{} markdown-link-check --config /workdir/.github/files/markdown.links.config.json {}' + @echo "Running markdown-link-check to validate URLs in the .md files" + docker run -it --rm -v ${PWD}:/workdir --entrypoint sh ghcr.io/tcort/markdown-link-check -c 'echo -e "/workdir/README.md\n$(find /workdir/docs/src -name "*.md")" | tr "\n" "\0" | xargs -0 -P 5 -n1 -I{} markdown-link-check --config /workdir/.github/files/markdown.links.config.json {}' # These steps automatically run before git push via a git hook [private] From 3117989f255f2061104179045e30e8a1c98a3254 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Mon, 4 Dec 2023 16:57:42 -0500 Subject: [PATCH 37/37] cleanup justfile --- justfile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/justfile b/justfile index 3b7a80c4e..7175bf892 100644 --- a/justfile +++ b/justfile @@ -255,11 +255,9 @@ lint: fmt clippy # Run cargo fmt fmt: cargo fmt --all -- --check - markdownlint-cli2 --config ".github/files/config.markdownlint-cli2.jsonc" --fix -# Run markdown lintting +# Reformat markdown files using markdownlint-cli2 fmt-md: - @echo "Running markdownlint-cli2..." docker run -it --rm -v $PWD:/workdir davidanson/markdownlint-cli2 --config /workdir/.github/files/config.markdownlint-cli2.jsonc --fix # Run Nightly cargo fmt, ordering imports @@ -271,10 +269,10 @@ clippy: cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings RUSTDOCFLAGS="-D warnings" cargo doc --no-deps --workspace -# Run markdown links checking +# Validate markdown URLs with markdown-link-check clippy-md: - @echo "Running markdown-link-check to validate URLs in the .md files" - docker run -it --rm -v ${PWD}:/workdir --entrypoint sh ghcr.io/tcort/markdown-link-check -c 'echo -e "/workdir/README.md\n$(find /workdir/docs/src -name "*.md")" | tr "\n" "\0" | xargs -0 -P 5 -n1 -I{} markdown-link-check --config /workdir/.github/files/markdown.links.config.json {}' + docker run -it --rm -v ${PWD}:/workdir --entrypoint sh ghcr.io/tcort/markdown-link-check -c \ + 'echo -e "/workdir/README.md\n$(find /workdir/docs/src -name "*.md")" | tr "\n" "\0" | xargs -0 -P 5 -n1 -I{} markdown-link-check --config /workdir/.github/files/markdown.links.config.json {}' # These steps automatically run before git push via a git hook [private]