Amazon service extensions

- Refactored DynamoDB extension
- Added S3 client extension

Author: marcinczeczko

azure-pipelines.yml

@@ -224,11 +224,11 @@ stages:
         parameters:
           timeoutInMinutes: 15
           modules:
-            - amazon-dynamodb
+            - amazon-services
             - amazon-lambda
             - amazon-lambda-http-it
           name: amazon
-          dynamodb: true
+          amazonStack: true
 
       - template: ci-templates/native-build-steps.yaml
         parameters:

bom/deployment/pom.xml

@@ -401,11 +401,21 @@
                 <artifactId>quarkus-amazon-lambda-resteasy-deployment</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.quarkus</groupId>
+                <artifactId>quarkus-amazon-common-deployment</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>io.quarkus</groupId>
                 <artifactId>quarkus-amazon-dynamodb-deployment</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.quarkus</groupId>
+                <artifactId>quarkus-amazon-s3-deployment</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>io.quarkus</groupId>
                 <artifactId>quarkus-amazon-lambda-http-deployment</artifactId>

bom/runtime/pom.xml

@@ -626,11 +626,23 @@
                 <artifactId>quarkus-amazon-lambda-http</artifactId>
                 <version>${project.version}</version>
             </dependency>
+
+
+            <dependency>
+                <groupId>io.quarkus</groupId>
+                <artifactId>quarkus-amazon-common</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>io.quarkus</groupId>
                 <artifactId>quarkus-amazon-dynamodb</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.quarkus</groupId>
+                <artifactId>quarkus-amazon-s3</artifactId>
+                <version>${project.version}</version>
+            </dependency>
             <dependency>
                 <groupId>io.quarkus</groupId>
                 <artifactId>quarkus-azure-functions-http</artifactId>
@@ -2215,16 +2227,52 @@
                 <version>${aws-lambda-serverless-java-container.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>sdk-core</artifactId>
+                <version>${awssdk.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>aws-core</artifactId>
+                <version>${awssdk.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>regions</artifactId>
+                <version>${awssdk.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>auth</artifactId>
+                <version>${awssdk.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>http-client-spi</artifactId>
+                <version>${awssdk.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
                 <artifactId>dynamodb</artifactId>
                 <version>${awssdk.version}</version>
             </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>s3</artifactId>
+                <version>${awssdk.version}</version>
+            </dependency>
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
                 <artifactId>netty-nio-client</artifactId>
                 <version>${awssdk.version}</version>
             </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>url-connection-client</artifactId>
+                <version>${awssdk.version}</version>
+            </dependency>
             <dependency>
                 <groupId>software.amazon.awssdk</groupId>
                 <artifactId>apache-client</artifactId>

ci-templates/jvm-build-steps.yaml

@@ -12,8 +12,8 @@ steps:
 - script: docker run --rm --publish 5432:5432 --name build-postgres -e POSTGRES_USER=hibernate_orm_test -e POSTGRES_PASSWORD=hibernate_orm_test -e POSTGRES_DB=hibernate_orm_test -d postgres:10.5
   displayName: 'start postgres'
 
-- script: docker run --rm --publish 8000:8000 --name build-dynamodb -d amazon/dynamodb-local:1.11.477
-  displayName: 'start dynamodb'
+- script: docker run --rm --publish 8000:4569 --publish 8008:4572 --name build-amazon-service-clients -e SERVICES=s3,dynamodb -e START_WEB=0 -d localstack/localstack
+  displayName: 'start aws local stack'
 
 - bash: |
     sudo apt-get update &&
@@ -28,5 +28,5 @@ steps:
     goals: 'install'
     mavenOptions: $(MAVEN_OPTS)
     jdkVersionOption: ${{ parameters.jdk }}
-    options: '-B --settings azure-mvn-settings.xml -Dnative-image.docker-build -Dtest-postgresql -Dtest-elasticsearch -Dtest-mysql -Dtest-dynamodb -Dtest-vault -Dno-format'
+    options: '-B --settings azure-mvn-settings.xml -Dnative-image.docker-build -Dtest-postgresql -Dtest-elasticsearch -Dtest-mysql -Dtest-dynamodb -Dtest-s3 -Dtest-vault -Dno-format'
 

ci-templates/native-build-steps.yaml

@@ -25,9 +25,9 @@ jobs:
     - ${{ if eq(parameters.postgres, 'true') }}:
       - script: docker run --rm --publish 5432:5432 --name build-postgres -e POSTGRES_USER=hibernate_orm_test -e POSTGRES_PASSWORD=hibernate_orm_test -e POSTGRES_DB=hibernate_orm_test -d postgres:10.5
         displayName: 'start postgres'
-    - ${{ if eq(parameters.dynamodb, 'true') }}:
-      - script: docker run --rm --publish 8000:8000 --name build-dynamodb -d amazon/dynamodb-local:1.11.477
-        displayName: 'start dynamodb'
+    - ${{ if eq(parameters.amazonStack, 'true') }}:
+      - script: docker run --rm --publish 8000:4569 --publish 8008:4572 --name build-amazon-service-clients -e SERVICES=s3,dynamodb -e START_WEB=0 -d localstack/localstack
+        displayName: 'start aws local stack'
     - ${{ if eq(parameters.keycloak, 'true') }}:
         - script: docker run --rm --publish 8180:8080 --name build-keycloak  -e KEYCLOAK_USER=admin -e KEYCLOAK_PASSWORD=admin -e JAVA_OPTS="-server -Xms64m -Xmx512m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dkeycloak.profile.feature.upload_scripts=enabled" -d quay.io/keycloak/keycloak
           displayName: 'start keycloak'

docs/src/main/asciidoc/dynamodb-guide.adoc renamed to docs/src/main/asciidoc/amazon-dynamodb-guide.adoc

@@ -19,7 +19,7 @@ Keep in mind it's actively developed and does not support yet all the features a
 
 The Quarkus extension supports two programming models:
 
-* Blocking access using the Apache HTTP Client
+* Blocking access using URL Connection HTTP client (by default) or the Apache HTTP Client
 * https://docs.aws.amazon.com/sdk-for-java/v2/developer-guide/basics-async.html[Asynchronous programming] based on JDK's `CompletableFuture` objects and the Netty HTTP client.
 
 In this guide, we see how you can get your REST services to use the DynamoDB locally and on AWS.
@@ -99,15 +99,15 @@ First, we need a new project. Create a new project with the following command:
 [source,shell,subs=attributes+]
 ----
 mvn io.quarkus:quarkus-maven-plugin:{quarkus-version}:create \
-    -DprojectGroupId=org.acme \
+    -DprojectGroupId=org.acme.amazon \
     -DprojectArtifactId=dynamodb-client \
-    -DclassName="org.acme.dynamodb.FruitResource" \
+    -DclassName="org.acme.amazon.dynamodb.FruitResource" \
     -Dpath="/fruits" \
     -Dextensions="resteasy-jsonb,dynamodb"
 ----
 
 This command generates a Maven structure importing the RESTEasy/JAX-RS and DynamoDB Client extensions.
-After this, the `amazon-dynamodb` extension has been added to your `pom.xml`.
+After this, the `dynamodb` extension has been added to your `pom.xml`.
 
 == Creating JSON REST service
 
@@ -117,7 +117,7 @@ First, let's create the `Fruit` bean as follows:
 
 [source,java]
 ----
-package org.acme.dynamodb;
+package org.acme.amazon.dynamodb;
 
 import java.util.Map;
 import java.util.Objects;
@@ -180,11 +180,11 @@ public class Fruit {
 Nothing fancy. One important thing to note is that having a default constructor is required by the JSON serialization layer. The static `from` method creates a bean based on the `Map`
 object provided by the DynamoDB client response.
 
-Now create a `org.acme.dynamodb.AbstractService` that will consist of helper methods that prepare DynamoDB request objects for reading and adding items to the table.
+Now create a `org.acme.amazon.dynamodb.AbstractService` that will consist of helper methods that prepare DynamoDB request objects for reading and adding items to the table.
 
 [source,java]
 ----
-package org.acme.dynamodb;
+package org.acme.amazon.dynamodb;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -232,11 +232,11 @@ public abstract class AbstractService {
 }
 ----
 
-Then, create a `org.acme.dynamodb.FruitSyncService` that will be the business layer of our application and stores/loads the fruits from DynamoDB using the synchronous client.
+Then, create a `org.acme.amazon.dynamodb.FruitSyncService` that will be the business layer of our application and stores/loads the fruits from DynamoDB using the synchronous client.
 
 [source,java]
 ----
-package org.acme.dynamodb;
+package org.acme.amazon.dynamodb;
 
 import java.util.List;
 import java.util.stream.Collectors;
@@ -269,7 +269,7 @@ public class FruitSyncService extends AbstractService {
 }
 ----
 
-Now, edit the `org.acme.dynamodb.FruitResource` class as follows:
+Now, edit the `org.acme.amazon.dynamodb.FruitResource` class as follows:
 
 [source,java]
 ----
@@ -318,32 +318,64 @@ The implementation is pretty straightforward and you just need to define your en
 == Configuring DynamoDB clients
 
 Both DynamoDB clients (sync and async) are configurable via the `application.properties` file that can be provided in the `src/main/resources` directory.
+Additionally, you need to add to the classpath a proper implementation of the sync client. By default the extension uses URL connection HTTP client, so
+add a URL connection client dependency to the `pom.xml` file:
+
+[source,xml]
+----
+<dependency>
+    <groupId>software.amazon.awssdk</groupId>
+    <artifactId>url-connection-client</artifactId>
+</dependency>
+----
+
+If you want to use Apache HTTP client instead, configure it as follows:
+[source,properties]
+----
+quarkus.amazon.sync-client.dynamodb.type=apache
+----
+
+And add following dependency to the application `pom.xml`:
+[source,xml]
+----
+<dependency>
+    <groupId>software.amazon.awssdk</groupId>
+    <artifactId>apache-client</artifactId>
+    <exclusions>
+        <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+        </exclusion>
+    </exclusions>
+</dependency>
+----
+NOTE: It's important to exclude `commons-logging` from the client dependency to force Apache HTTP client to use Quarkus logger.
 
 If you're going to use a local DynamoDB instance, configure it as follows:
 
 [source,properties]
 ----
-quarkus.dynamodb.endpoint-override=http://localhost:8000
+quarkus.amazon.dynamodb.endpoint-override=http://localhost:8000
 
-quarkus.dynamodb.aws.region=eu-central-1
-quarkus.dynamodb.aws.credentials.type=STATIC
-quarkus.dynamodb.aws.credentials.static-provider.access-key-id=test-key
-quarkus.dynamodb.aws.credentials.static-provider.secret-access-key=test-secret
+quarkus.amazon.aws.dynamodb.region=eu-central-1
+quarkus.amazon.aws.dynamodb.credentials.type=static
+quarkus.amazon.aws.dynamodb.credentials.static-provider.access-key-id=test-key
+quarkus.amazon.aws.dynamodb.credentials.static-provider.secret-access-key=test-secret
 ----
 
-- `quarkus.dynamodb.aws.region` - It's required by the client, but since you're using a local DynamoDB instance you can pick any valid AWS region.
-- `quarkus.dynamodb.aws.credentials.type` - Set `STATIC` credentials provider with any values for `access-key-id` and `secret-access-key`
-- `quarkus.dynamodb.endpoint-override` - Override the DynamoDB client to use a local instance instead of an AWS service
+- `quarkus.amazon.aws.dynamodb.region` - It's required by the client, but since you're using a local DynamoDB instance you can pick any valid AWS region.
+- `quarkus.amazon.aws.dynamodb.credentials.type` - Set `static` credentials provider with any values for `access-key-id` and `secret-access-key`
+- `quarkus.amazon.dynamodb.endpoint-override` - Override the DynamoDB client to use a local instance instead of an AWS service
 
 If you want to work with an AWS account, you'd need to set it with:
 [source,properties]
 ----
-quarkus.dynamodb.aws.region=<YOUR_REGION>
-quarkus.dynamodb.aws.credentials.type=DEFAULT
+quarkus.amazon.aws.dynamodb.region=<YOUR_REGION>
+quarkus.amazon.aws.dynamodb.credentials.type=default
 ----
 
-- `quarkus.dynamodb.aws.region` you should set it to the region where you provisioned the DynamoDB table,
-- `quarkus.dynamodb.aws.credentials.type` - use the `DEFAULT` credentials provider chain that looks for credentials in this order:
+- `quarkus.amazon.aws.dynamodb.region` you should set it to the region where you provisioned the DynamoDB table,
+- `quarkus.amazon.aws.dynamodb.credentials.type` - use the `default` credentials provider chain that looks for credentials in this order:
 - Java System Properties - `aws.accessKeyId` and `aws.secretKey`
 * Environment Variables - `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
 * Credential profiles file at the default location (`~/.aws/credentials`) shared by all AWS SDKs and the AWS CLI
@@ -364,11 +396,11 @@ Depending on your system, that will take some time.
 
 Thanks to the AWS SDK v2.x used by the Quarkus extension, you can use the asynchronous programming model out of the box.
 
-Create a `org.acme.dynamodb.FruitAsyncService` that will be similar to our `FruitSyncService` but using an asynchronous programming model.
+Create a `org.acme.amazon.dynamodb.FruitAsyncService` that will be similar to our `FruitSyncService` but using an asynchronous programming model.
 
 [source,java]
 ----
-package org.acme.dynamodb;
+package org.acme.amazon.dynamodb;
 
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
@@ -400,11 +432,11 @@ public class FruitAsyncService extends AbstractService {
 }
 ----
 
-And an asynchronous REST resource:
+Create an asynchronous REST resource:
 
 [source,java]
 ----
-package org.acme.dynamodb;
+package org.acme.amazon.dynamodb;
 
 import java.util.List;
 import java.util.concurrent.CompletionStage;
@@ -445,7 +477,22 @@ public class FruitAsyncResource {
 }
 ----
 
+And add Netty HTTP client dependency to the `pom.xml`:
+
+[source,xml]
+----
+<dependency>
+    <groupId>software.amazon.awssdk</groupId>
+    <artifactId>netty-nio-client</artifactId>
+</dependency>
+----
+
 == Configuration Reference
 
+=== DynamoDB service configuration reference
+
 include::{generated-dir}/config/quarkus-dynamodb.adoc[opts=optional, leveloffset=+1]
 
+=== Amazon services common configuration reference
+
+include::{generated-dir}/config/quarkus-amazon.adoc[opts=optional, leveloffset=+1]