Skip to content
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
25adc30
clean up `is_url`
CommanderStorm May 24, 2025
ae9b2df
implement force_path_style and no_credentials via the config file as …
CommanderStorm May 24, 2025
38418fd
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 24, 2025
d30c4f9
change the env-var-access to be case-agnostic
CommanderStorm May 24, 2025
39a3a38
fix formatting
CommanderStorm May 24, 2025
8f15639
remove accidental Cargo.toml change
CommanderStorm May 24, 2025
75d85ba
reword docs slightly
CommanderStorm May 26, 2025
fc0aec4
Update martin/src/pmtiles/mod.rs
CommanderStorm May 26, 2025
681b2fa
add `get_env_as_bool`
CommanderStorm May 26, 2025
06ccffe
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 26, 2025
83a5afa
Change the env var docs to use above shema too
CommanderStorm May 26, 2025
f429fde
change styling somewhat
CommanderStorm May 26, 2025
2a95f71
Merge branch 'main' into s3-followup
CommanderStorm May 26, 2025
d214ae6
fix gramar
CommanderStorm May 26, 2025
397c5f1
change impl of `get_env_as_bool` for aestetical reasons
CommanderStorm May 26, 2025
bb72af7
remove matches and relace it with a comparison
CommanderStorm May 26, 2025
a8e05d1
change to using `require_credentials`
CommanderStorm May 27, 2025
6123d37
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 27, 2025
a16a3d0
fix typo
CommanderStorm May 27, 2025
95029d6
change remaining instaces of `AWS_REQUIRE_CREDENTIALS`
CommanderStorm May 27, 2025
65b5380
add comment about why `AWS_NO_CREDENTIALS` exists
CommanderStorm May 27, 2025
7cb030e
make it clearer how environment variables play into martin
CommanderStorm May 27, 2025
f51590d
make sure that the `AWS_NO_CREDENTIALS`-handling is explicite
CommanderStorm May 27, 2025
5bf4280
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 27, 2025
f46b359
fix typo
CommanderStorm May 27, 2025
8153b7e
change to using `AWS_SKIP_CREDENTIALS`
CommanderStorm May 27, 2025
e6e50a7
fix an temporary value dropped while borrowed issue
CommanderStorm May 27, 2025
9fb3f0d
change from match statements to unwrap_or, because clippy
CommanderStorm May 27, 2025
c5d98d1
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] May 27, 2025
d89fd16
Merge branch 'main' into s3-followup
CommanderStorm May 27, 2025
63a8252
Merge branch 'main' into s3-followup
CommanderStorm May 29, 2025
1e29b05
fix minor typing fix regarding punctuation
CommanderStorm May 29, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/src/config-file.md
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,12 @@ postgres:

# Publish PMTiles files from local disk or proxy to a web server
pmtiles:
# allows forcing path style URLs for S3 buckets [default: false]
#
# A path style URL is a URL that uses the bucket name as part of the path (`mys3.com/somebucket`) instead of the hostname (`somebucket.mys3.com`).
force_path_style: false
# To send requests anonymously for publicly available buckets [default: false]
no_credentials: false
paths:
# scan this whole dir, matching all *.pmtiles files
- /dir-path
Expand Down
18 changes: 16 additions & 2 deletions docs/src/env-vars.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
## Environment Variables

You can also configure Martin using environment variables, but only if the configuration file is not used. See [configuration section](config-file.md) on how to use environment variables with config files. See also [SSL configuration](pg-connections.md#postgresql-ssl-connections) section below.
You can also configure Martin using environment variables, but only if the configuration file is not used.
See [configuration section](config-file.md) on how to use environment variables with config files.
See also [SSL configuration](pg-connections.md#postgresql-ssl-connections) section below.

| Environment var <br/> Config File key | Example | Description |
|------------------------------------------|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
Expand All @@ -9,4 +11,16 @@ You can also configure Martin using environment variables, but only if the confi
| `PGSSLCERT` <br/> `ssl_cert` | `./postgresql.crt` | A file with a client SSL certificate. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLCERT) |
| `PGSSLKEY` <br/> `ssl_key` | `./postgresql.key` | A file with the key for the client SSL certificate. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLKEY) |
| `PGSSLROOTCERT` <br/> `ssl_root_cert` | `./root.crt` | A file with trusted root certificate(s). The file should contain a sequence of PEM-formatted CA certificates. [docs](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNECT-SSLROOTCERT) |
| `AWS_LAMBDA_RUNTIME_API` | | If defined, connect to AWS Lambda to handle requests. The regular HTTP server is not used. See [Running in AWS Lambda](run-with-lambda.md) |
| `AWS_LAMBDA_RUNTIME_API` <br/> - | | If defined, connect to AWS Lambda to handle requests. The regular HTTP server is not used. See [Running in AWS Lambda](run-with-lambda.md) |

To [access PMTiles via S3](sources-files.md#serving-pmtiles-via-s3), we also support the following configuration options:

| Environment var <br/> Config File key | Example | Description |
| ---------------------------------------------------------- | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `AWS_ACCESS_KEY_ID` <br/> - | `AKIAEXAMPLE12345678` | AWS access key ID used for authenticating requests when using long-term or temporary credentials. |
| `AWS_SECRET_ACCESS_KEY` <br/> - | `wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY` | AWS secret access key paired with the access key ID. |
| `AWS_SESSION_TOKEN` <br/> - | `FwoGZXIvYXdzEF0aD...` | Session token used with temporary security credentials (e.g., from AWS STS). Required if you're using `AssumeRole`. |
| `AWS_PROFILE` <br/> - | `default` | Specifies which named profile to use from the AWS credentials/config files. |
| `AWS_REGION` <br/> - | `us-west-2` | Sets the AWS region to send requests to, e.g., `us-east-1`, `eu-central-1`. |
| `AWS_NO_CREDENTIALS` <br/> `pmtiles.no_credentials` | `true` | Disable credential loading and to send requests anonymously for publicly available buckets. |
| `AWS_S3_FORCE_PATH_STYLE` <br/> `pmtiles.force_path_style` | `true` | Forces the AWS SDK to use path-style URLs for S3 like `s3.amazonaws.com/bucket/key`) instead of virtual-hosted style. Useful for local S3-compatible services like MinIO. |
27 changes: 23 additions & 4 deletions docs/src/sources-files.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@ martin /path/to/mbtiles/file.mbtiles /path/to/directory https://example.org/
You may also want to generate a [config file](config-file.md) using the `--save-config my-config.yaml`, and later edit
it and use it with `--config my-config.yaml` option.

## PMTiles S3
### Serving PMTiles via S3

#### Authentication with AWS credentials

Martin supports authenticated S3 sources using environment variables.

Expand All @@ -23,15 +25,32 @@ By default, Martin will use default profile's credentials unless these [AWS envi
- `AWS_PROFILE` - to specify profile instead of access key variables
- `AWS_REGION` - if set, must match the region of the bucket in the S3 URI

### Anonymous credentials
#### Anonymous credentials

To send requests anonymously for publicly available buckets, set the environment variable `AWS_NO_CREDENTIALS=1` or configuration key `no_credentials: true` respectively.

Note: you still need to set `AWS_REGION` to the correct region.

Example configuration:

```yaml
pmtiles:
no_credentials: true
sources:
tiles: s3://bucket/path/to/tiles.pmtiles
```

#### Url styles

To send requests anonymously for publicly available buckets, set the environment variable `AWS_NO_CREDENTIALS=1`.
Note that you still need to set `AWS_REGION` to the correct region.
We also support forcing path style URLs for S3 buckets via the environment variable `AWS_S3_FORCE_PATH_STYLE=1` or configuration key `force_path_style: true`.
This allows you to use this functionality for [`MinIO`](https://min.io/) or similar s3-compatible instances which use path style URLs.
A path style URL is a URL that uses the bucket name as part of the path (`mys3.com/somebucket`) instead of the hostname (`somebucket.mys3.com`).

Example configuration:

```yaml
pmtiles:
force_path_style: true
sources:
tiles: s3://bucket/path/to/tiles.pmtiles
```
34 changes: 17 additions & 17 deletions martin/src/args/root.rs
Original file line number Diff line number Diff line change
Expand Up @@ -154,26 +154,26 @@ impl Args {
}
}

/// Check if a string is a valid [`url::Url`] with a specified extension.
#[cfg(any(feature = "pmtiles", feature = "mbtiles", feature = "cog"))]
fn is_url(s: &str, extension: &[&str]) -> bool {
if s.starts_with("http") || s.starts_with("s3") {
if let Ok(url) = url::Url::parse(s) {
if url.scheme() == "s3" {
return url.path().split('/').any(|segment| {
segment
.rsplit('.')
.next()
.is_some_and(|ext| extension.contains(&ext))
});
}
if ["http", "https"].contains(&url.scheme()) {
if let Some(ext) = url.path().rsplit('.').next() {
return extension.contains(&ext);
}
}
}
let Ok(url) = url::Url::parse(s) else {
return false;
};
match url.scheme() {
"s3" => url.path().split('/').any(|segment| {
segment
.rsplit('.')
.next()
.is_some_and(|ext| extension.contains(&ext))
}),
"http" | "https" => url
.path()
.rsplit('.')
.next()
.is_some_and(|ext| extension.contains(&ext)),
_ => false,
}
false
}

#[cfg(any(
Expand Down
51 changes: 45 additions & 6 deletions martin/src/pmtiles/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,14 @@ impl DirectoryCache for PmtCache {
#[serde_with::skip_serializing_none]
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct PmtConfig {
/// force path style URLs for S3 buckets
///
/// A path style URL is a URL that uses the bucket name as part of the path like `mys3.com/somebucket` instead of the hostname `somebucket.mys3.com`.
#[serde(default, alias = "aws_s3_force_path_style")]
pub force_path_style: Option<bool>,
/// send requests anonymously for publicly available buckets
#[serde(default, alias = "aws_no_credentials")]
pub no_credentials: Option<bool>,
#[serde(flatten)]
pub unrecognized: UnrecognizedValues,

Expand Down Expand Up @@ -146,9 +154,24 @@ impl SourceConfigExtras for PmtConfig {

async fn new_sources_url(&self, id: String, url: Url) -> FileResult<TileInfoSource> {
match url.scheme() {
"s3" => Ok(Box::new(
PmtS3Source::new(self.new_cached_source(), id, url).await?,
)),
"s3" => {
let force_path_style = self
.force_path_style
.unwrap_or_else(|| get_env_as_bool("AWS_S3_FORCE_PATH_STYLE"));
let no_credentials = self
.no_credentials
.unwrap_or_else(|| get_env_as_bool("AWS_NO_CREDENTIALS"));
Ok(Box::new(
PmtS3Source::new(
self.new_cached_source(),
id,
url,
no_credentials,
force_path_style,
)
.await?,
))
}
_ => Ok(Box::new(
PmtHttpSource::new(
self.client.clone().unwrap(),
Expand Down Expand Up @@ -307,15 +330,21 @@ impl PmtHttpSource {
impl_pmtiles_source!(PmtS3Source, AwsS3Backend, Url, identity, InvalidUrlMetadata);

impl PmtS3Source {
pub async fn new(cache: PmtCache, id: String, url: Url) -> FileResult<Self> {
pub async fn new(
cache: PmtCache,
id: String,
url: Url,
no_credentials: bool,
force_path_style: bool,
) -> FileResult<Self> {
let mut aws_config_builder = aws_config::from_env();
if std::env::var("AWS_NO_CREDENTIALS").unwrap_or_default() == "1" {
if no_credentials {
aws_config_builder = aws_config_builder.no_credentials();
}
let aws_config = aws_config_builder.load().await;

let s3_config = S3ConfigBuilder::from(&aws_config)
.force_path_style(std::env::var("AWS_S3_FORCE_PATH_STYLE").unwrap_or_default() == "1")
.force_path_style(force_path_style)
.build();
let client = S3Client::from_conf(s3_config);

Expand Down Expand Up @@ -361,3 +390,13 @@ impl PmtFileSource {
Self::new_int(id, path, reader).await
}
}

/// Interpret an environment variable as a [`bool`]
///
/// This ignores casing, but interprets bad utf8 encoding as `false`.
fn get_env_as_bool(key: &'static str) -> bool {
let env_val = std::env::var_os(key)
.unwrap_or_default()
.to_ascii_lowercase();
[Some("1"), Some("true")].contains(&env_val.to_str())
}