Merge pull request #1158 from tskinn/add-dynamodb-provider

add dynamodb backend
This commit is contained in:
Emile Vauge 2017-03-17 18:02:02 +01:00 committed by GitHub
commit 677899d9ff
25 changed files with 11312 additions and 4 deletions

View file

@ -13,7 +13,7 @@
Træfɪk (pronounced like [traffic](https://speak-ipa.bearbin.net/speak.cgi?speak=%CB%88tr%C3%A6f%C9%AAk)) is a modern HTTP reverse proxy and load balancer made to deploy microservices with ease.
It supports several backends ([Docker](https://www.docker.com/), [Swarm](https://docs.docker.com/swarm), [Kubernetes](http://kubernetes.io), [Marathon](https://mesosphere.github.io/marathon/), [Mesos](https://github.com/apache/mesos), [Consul](https://www.consul.io/), [Etcd](https://coreos.com/etcd/), [Zookeeper](https://zookeeper.apache.org), [BoltDB](https://github.com/boltdb/bolt), [Eureka](https://github.com/Netflix/eureka), Rest API, file...) to manage its configuration automatically and dynamically.
It supports several backends ([Docker](https://www.docker.com/), [Swarm](https://docs.docker.com/swarm), [Kubernetes](http://kubernetes.io), [Marathon](https://mesosphere.github.io/marathon/), [Mesos](https://github.com/apache/mesos), [Consul](https://www.consul.io/), [Etcd](https://coreos.com/etcd/), [Zookeeper](https://zookeeper.apache.org), [BoltDB](https://github.com/boltdb/bolt), [Eureka](https://github.com/Netflix/eureka), [Amazon DynamoDB](https://aws.amazon.com/dynamodb/), Rest API, file...) to manage its configuration automatically and dynamically.
## Overview

View file

@ -52,6 +52,7 @@ type GlobalConfiguration struct {
Eureka *provider.Eureka `description:"Enable Eureka backend"`
ECS *provider.ECS `description:"Enable ECS backend"`
Rancher *provider.Rancher `description:"Enable Rancher backend"`
DynamoDB *provider.DynamoDB `description:"Enable DynamoDB backend"`
}
// DefaultEntryPoints holds default entry points
@ -406,6 +407,13 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration {
defaultRancher.Watch = true
defaultRancher.ExposedByDefault = true
// default DynamoDB
var defaultDynamoDB provider.DynamoDB
defaultDynamoDB.Constraints = types.Constraints{}
defaultDynamoDB.RefreshSeconds = 15
defaultDynamoDB.TableName = "traefik"
defaultDynamoDB.Watch = true
defaultConfiguration := GlobalConfiguration{
Docker: &defaultDocker,
File: &defaultFile,
@ -420,6 +428,7 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration {
Mesos: &defaultMesos,
ECS: &defaultECS,
Rancher: &defaultRancher,
DynamoDB: &defaultDynamoDB,
Retry: &Retry{},
}

View file

@ -11,7 +11,7 @@
Træfɪk is a modern HTTP reverse proxy and load balancer made to deploy microservices with ease.
It supports several backends ([Docker](https://www.docker.com/), [Swarm](https://docs.docker.com/swarm), [Mesos/Marathon](https://mesosphere.github.io/marathon/), [Consul](https://www.consul.io/), [Etcd](https://coreos.com/etcd/), [Zookeeper](https://zookeeper.apache.org), [BoltDB](https://github.com/boltdb/bolt), [Amazon ECS](https://aws.amazon.com/ecs/), Rest API, file...) to manage its configuration automatically and dynamically.
It supports several backends ([Docker](https://www.docker.com/), [Swarm](https://docs.docker.com/swarm), [Mesos/Marathon](https://mesosphere.github.io/marathon/), [Consul](https://www.consul.io/), [Etcd](https://coreos.com/etcd/), [Zookeeper](https://zookeeper.apache.org), [BoltDB](https://github.com/boltdb/bolt), [Amazon ECS](https://aws.amazon.com/ecs/), [Amazon DynamoDB](https://aws.amazon.com/dynamodb/), Rest API, file...) to manage its configuration automatically and dynamically.
## Overview

View file

@ -1569,3 +1569,74 @@ Labels can be used on task containers to override default behaviour:
- `traefik.frontend.passHostHeader=true`: forward client `Host` header to the backend.
- `traefik.frontend.priority=10`: override default frontend priority
- `traefik.frontend.entryPoints=http,https`: assign this frontend to entry points `http` and `https`. Overrides `defaultEntryPoints`.
## DynamoDB backend
Træfɪk can be configured to use Amazon DynamoDB as a backend configuration:
```toml
################################################################
# DynamoDB configuration backend
################################################################
# Enable DynamoDB configuration backend
#
# Optional
#
[dynamodb]
# DyanmoDB Table Name
#
# Optional
#
TableName = "traefik"
# Enable watch DynamoDB changes
#
# Optional
#
Watch = true
# Polling interval (in seconds)
#
# Optional
#
RefreshSeconds = 15
# Region to use when connecting to AWS
#
# Required
#
# Region = "us-west-1"
# AccessKeyID to use when connecting to AWS
#
# Optional
#
# AccessKeyID = "abc"
# SecretAccessKey to use when connecting to AWS
#
# Optional
#
# SecretAccessKey = "123"
# Endpoint of local dynamodb instance for testing
#
# Optional
#
# Endpoint = "http://localhost:8080"
```
Items in the dynamodb table must have three attributes:
- 'id' : string
- The id is the primary key.
- 'name' : string
- The name is used as the name of the frontend or backend.
- 'frontend' or 'backend' : map
- This attribute's structure matches exactly the structure of a Frontend or Backend type in traefik. See types/types.go for details. The presence or absence of this attribute determines its type. So an item should never have both a 'frontend' and a 'backend' attribute.

8
glide.lock generated
View file

@ -1,5 +1,5 @@
hash: f2a0b9af55c8312762e4148bd876e5bd2451c240b407fbb6d4a5dbc56bf46c05
updated: 2017-03-03T10:21:14.720631882+01:00
hash: 9d7c36c335fe9106ec79cb86a3b3824c23b63d3fb3a3fb4c75bd6915f3afcdd4
updated: 2017-03-08T18:53:12.139107148-07:00
imports:
- name: bitbucket.org/ww/goautoneg
version: 75cd24fc2f2c2a2088577d12123ddee5f54e0675
@ -44,6 +44,10 @@ imports:
- private/protocol/restxml
- private/protocol/xml/xmlutil
- private/waiter
- service/dynamodb
- service/dynamodb/dynamodbattribute
- service/dynamodb/dynamodbiface
- service/dynamodbattribute
- service/ec2
- service/ecs
- service/route53

View file

@ -127,6 +127,9 @@ import:
- aws/endpoints
- aws/request
- aws/session
- service/dynamodb
- service/dynamodb/dynamodbiface
- service/dynamodbattribute
- service/ec2
- service/ecs
- package: cloud.google.com/go

View file

@ -0,0 +1,181 @@
package main
import (
"errors"
"io/ioutil"
"net/http"
"os"
"os/exec"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
"github.com/containous/traefik/integration/utils"
"github.com/containous/traefik/types"
"github.com/go-check/check"
checker "github.com/vdemeester/shakers"
)
type DynamoDBSuite struct {
BaseSuite
}
type DynamoDBItem struct {
ID string `dynamodbav:"id"`
Name string `dynamodbav:"name"`
}
type DynamoDBBackendItem struct {
DynamoDBItem
Backend types.Backend `dynamodbav:"backend"`
}
type DynamoDBFrontendItem struct {
DynamoDBItem
Frontend types.Frontend `dynamodbav:"frontend"`
}
func (s *DynamoDBSuite) SetUpSuite(c *check.C) {
s.createComposeProject(c, "dynamodb")
s.composeProject.Start(c)
dynamoURL := "http://" + s.composeProject.Container(c, "dynamo").NetworkSettings.IPAddress + ":8000"
config := &aws.Config{
Region: aws.String("us-east-1"),
Credentials: credentials.NewStaticCredentials("id", "secret", ""),
Endpoint: aws.String(dynamoURL),
}
var sess *session.Session
err := utils.Try(60*time.Second, func() error {
_, err := session.NewSession(config)
if err != nil {
return err
}
sess = session.New(config)
return nil
})
svc := dynamodb.New(sess)
// create dynamodb table
params := &dynamodb.CreateTableInput{
AttributeDefinitions: []*dynamodb.AttributeDefinition{
{
AttributeName: aws.String("id"),
AttributeType: aws.String("S"),
},
},
KeySchema: []*dynamodb.KeySchemaElement{
{
AttributeName: aws.String("id"),
KeyType: aws.String("HASH"),
},
},
ProvisionedThroughput: &dynamodb.ProvisionedThroughput{
ReadCapacityUnits: aws.Int64(1),
WriteCapacityUnits: aws.Int64(1),
},
TableName: aws.String("traefik"),
}
_, err = svc.CreateTable(params)
if err != nil {
c.Error(err)
return
}
// load config into dynamodb
whoami1 := "http://" + s.composeProject.Container(c, "whoami1").NetworkSettings.IPAddress + ":80"
whoami2 := "http://" + s.composeProject.Container(c, "whoami2").NetworkSettings.IPAddress + ":80"
whoami3 := "http://" + s.composeProject.Container(c, "whoami3").NetworkSettings.IPAddress + ":80"
backend := DynamoDBBackendItem{
Backend: types.Backend{
Servers: map[string]types.Server{
"whoami1": {
URL: whoami1,
},
"whoami2": {
URL: whoami2,
},
"whoami3": {
URL: whoami3,
},
},
},
DynamoDBItem: DynamoDBItem{
ID: "whoami_backend",
Name: "whoami",
},
}
frontend := DynamoDBFrontendItem{
Frontend: types.Frontend{
EntryPoints: []string{
"http",
},
Backend: "whoami",
Routes: map[string]types.Route{
"hostRule": {
Rule: "Host:test.traefik.io",
},
},
},
DynamoDBItem: DynamoDBItem{
ID: "whoami_frontend",
Name: "whoami",
},
}
backendAttributeValue, err := dynamodbattribute.MarshalMap(backend)
c.Assert(err, checker.IsNil)
frontendAttributeValue, err := dynamodbattribute.MarshalMap(frontend)
c.Assert(err, checker.IsNil)
putParams := &dynamodb.PutItemInput{
Item: backendAttributeValue,
TableName: aws.String("traefik"),
}
_, err = svc.PutItem(putParams)
c.Assert(err, checker.IsNil)
putParams = &dynamodb.PutItemInput{
Item: frontendAttributeValue,
TableName: aws.String("traefik"),
}
_, err = svc.PutItem(putParams)
c.Assert(err, checker.IsNil)
}
func (s *DynamoDBSuite) TestSimpleConfiguration(c *check.C) {
dynamoURL := "http://" + s.composeProject.Container(c, "dynamo").NetworkSettings.IPAddress + ":8000"
file := s.adaptFile(c, "fixtures/dynamodb/simple.toml", struct{ DynamoURL string }{dynamoURL})
defer os.Remove(file)
cmd := exec.Command(traefikBinary, "--configFile="+file)
err := cmd.Start()
c.Assert(err, checker.IsNil)
defer cmd.Process.Kill()
err = utils.TryRequest("http://127.0.0.1:8081/api/providers", 120*time.Second, func(res *http.Response) error {
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
if !strings.Contains(string(body), "Host:test.traefik.io") {
return errors.New("incorrect traefik config")
}
return nil
})
c.Assert(err, checker.IsNil)
client := &http.Client{}
req, err := http.NewRequest("GET", "http://127.0.0.1:8080", nil)
c.Assert(err, checker.IsNil)
req.Host = "test.traefik.io"
response, err := client.Do(req)
c.Assert(err, checker.IsNil)
c.Assert(response.StatusCode, checker.Equals, 200)
}
func (s *DynamoDBSuite) TearDownSuite(c *check.C) {
if s.composeProject != nil {
s.composeProject.Stop(c)
}
}

View file

@ -0,0 +1,16 @@
defaultEntryPoints = ["http"]
logLevel = "DEBUG"
[entryPoints]
[entryPoints.http]
address = ":8080"
[dynamodb]
AccessKeyID = "key"
SecretAccessKey = "secret"
Endpoint = "{{.DynamoURL}}"
Region = "us-east-1"
[web]
address = ":8081"

View file

@ -36,6 +36,7 @@ func init() {
check.Suite(&MesosSuite{})
check.Suite(&EurekaSuite{})
check.Suite(&AcmeSuite{})
check.Suite(&DynamoDBSuite{})
}
var traefikBinary = "../dist/traefik"

View file

@ -0,0 +1,16 @@
dynamo:
image: deangiberson/aws-dynamodb-local
command: -sharedDb
ports:
- "8000:8000"
expose:
- "8000"
whoami1:
image: emilevauge/whoami
whoami2:
image: emilevauge/whoami
whoami3:
image: emilevauge/whoami

207
provider/dynamodb.go Normal file
View file

@ -0,0 +1,207 @@
package provider
import (
"context"
"errors"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
"github.com/cenk/backoff"
"github.com/containous/traefik/job"
"github.com/containous/traefik/log"
"github.com/containous/traefik/safe"
"github.com/containous/traefik/types"
)
var _ Provider = (*DynamoDB)(nil)
// DynamoDB holds configuration for DynamoDB provider.
type DynamoDB struct {
BaseProvider `mapstructure:",squash"`
AccessKeyID string `description:"The AWS credentials access key to use for making requests"`
RefreshSeconds int `description:"Polling interval (in seconds)"`
Region string `description:"The AWS region to use for requests"`
SecretAccessKey string `description:"The AWS credentals secret key to use for making requests"`
TableName string `description:"The AWS dynamodb table that stores configuration for traefik"`
Endpoint string `description:"The endpoint of a dynamodb. Used for testing with a local dynamodb"`
}
type dynamoClient struct {
db dynamodbiface.DynamoDBAPI
}
// createClient configures aws credentials and creates a dynamoClient
func (provider *DynamoDB) createClient() (*dynamoClient, error) {
log.Infof("Creating DynamoDB client...")
sess := session.New()
if provider.Region == "" {
return nil, errors.New("no Region provided for DynamoDB")
}
cfg := &aws.Config{
Region: &provider.Region,
Credentials: credentials.NewChainCredentials(
[]credentials.Provider{
&credentials.StaticProvider{
Value: credentials.Value{
AccessKeyID: provider.AccessKeyID,
SecretAccessKey: provider.SecretAccessKey,
},
},
&credentials.EnvProvider{},
&credentials.SharedCredentialsProvider{},
defaults.RemoteCredProvider(*(defaults.Config()), defaults.Handlers()),
}),
}
if provider.Endpoint != "" {
cfg.Endpoint = aws.String(provider.Endpoint)
}
return &dynamoClient{
dynamodb.New(sess, cfg),
}, nil
}
// scanTable scans the given table and returns slice of all items in the table
func (provider *DynamoDB) scanTable(client *dynamoClient) ([]map[string]*dynamodb.AttributeValue, error) {
log.Debugf("Scanning DynamoDB table: %s ...", provider.TableName)
params := &dynamodb.ScanInput{
TableName: aws.String(provider.TableName),
}
items := make([]map[string]*dynamodb.AttributeValue, 0)
err := client.db.ScanPages(params,
func(page *dynamodb.ScanOutput, lastPage bool) bool {
items = append(items, page.Items...)
return !lastPage
})
if err != nil {
log.Errorf("Failed to scan DynamoDB table %s", provider.TableName)
return nil, err
}
log.Debugf("Successfully scanned DynamoDB table %s", provider.TableName)
return items, nil
}
// loadDynamoConfig retrieves items from dynamodb and converts them into Backends and Frontends in a Configuration
func (provider *DynamoDB) loadDynamoConfig(client *dynamoClient) (*types.Configuration, error) {
items, err := provider.scanTable(client)
if err != nil {
return nil, err
}
log.Debugf("Number of Items retrieved from DynamoDB: %d", len(items))
backends := make(map[string]*types.Backend)
frontends := make(map[string]*types.Frontend)
// unmarshal dynamoAttributes into Backends and Frontends
for i, item := range items {
log.Debugf("DynamoDB Item: %d\n%v", i, item)
// verify the type of each item by checking to see if it has
// the corresponding type, backend or frontend map
if backend, exists := item["backend"]; exists {
log.Debugf("Unmarshaling backend from DynamoDB...")
tmpBackend := &types.Backend{}
err = dynamodbattribute.Unmarshal(backend, tmpBackend)
if err != nil {
log.Errorf(err.Error())
} else {
backends[*item["name"].S] = tmpBackend
log.Debugf("Backend from DynamoDB unmarshalled successfully")
}
} else if frontend, exists := item["frontend"]; exists {
log.Debugf("Unmarshaling frontend from DynamoDB...")
tmpFrontend := &types.Frontend{}
err = dynamodbattribute.Unmarshal(frontend, tmpFrontend)
if err != nil {
log.Errorf(err.Error())
} else {
frontends[*item["name"].S] = tmpFrontend
log.Debugf("Frontend from DynamoDB unmarshalled successfully")
}
} else {
log.Warnf("Error in format of DynamoDB Item: %v", item)
}
}
return &types.Configuration{
Backends: backends,
Frontends: frontends,
}, nil
}
// Provide provides the configuration to traefik via the configuration channel
// if watch is enabled it polls dynamodb
func (provider *DynamoDB) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
log.Debugf("Providing DynamoDB...")
provider.Constraints = append(provider.Constraints, constraints...)
handleCanceled := func(ctx context.Context, err error) error {
if ctx.Err() == context.Canceled || err == context.Canceled {
return nil
}
return err
}
pool.Go(func(stop chan bool) {
ctx, cancel := context.WithCancel(context.Background())
go func() {
select {
case <-stop:
cancel()
}
}()
operation := func() error {
aws, err := provider.createClient()
if err != nil {
return handleCanceled(ctx, err)
}
configuration, err := provider.loadDynamoConfig(aws)
if err != nil {
return handleCanceled(ctx, err)
}
configurationChan <- types.ConfigMessage{
ProviderName: "dynamodb",
Configuration: configuration,
}
if provider.Watch {
reload := time.NewTicker(time.Second * time.Duration(provider.RefreshSeconds))
defer reload.Stop()
for {
log.Debugf("Watching DynamoDB...")
select {
case <-reload.C:
configuration, err := provider.loadDynamoConfig(aws)
if err != nil {
return handleCanceled(ctx, err)
}
configurationChan <- types.ConfigMessage{
ProviderName: "dynamodb",
Configuration: configuration,
}
case <-ctx.Done():
return handleCanceled(ctx, ctx.Err())
}
}
}
return nil
}
notify := func(err error, time time.Duration) {
log.Errorf("DynamoDB error: %s time: %v", err.Error(), time)
}
err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
if err != nil {
log.Errorf("Failed to connect to DynamoDB. %s", err.Error())
}
})
return nil
}

140
provider/dynamodb_test.go Normal file
View file

@ -0,0 +1,140 @@
package provider
import (
"errors"
"reflect"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
"github.com/containous/traefik/types"
)
type mockDynamoDBCLient struct {
dynamodbiface.DynamoDBAPI
testWithError bool
}
var backend = &types.Backend{
HealthCheck: &types.HealthCheck{
URL: "/build",
},
Servers: map[string]types.Server{
"server1": {
URL: "http://test.traefik.io",
},
},
}
var frontend = &types.Frontend{
EntryPoints: []string{"http"},
Backend: "test.traefik.io",
Routes: map[string]types.Route{
"route1": {
Rule: "Host:test.traefik.io",
},
},
}
// ScanPages simulates a call to ScanPages (see https://docs.aws.amazon.com/sdk-for-go/api/service/dynamodb/#DynamoDB.ScanPages)
// by running the fn function twice and returning an item each time.
func (m *mockDynamoDBCLient) ScanPages(input *dynamodb.ScanInput, fn func(*dynamodb.ScanOutput, bool) bool) error {
if m.testWithError {
return errors.New("fake error")
}
attributeBackend, err := dynamodbattribute.Marshal(backend)
if err != nil {
return err
}
attributeFrontend, err := dynamodbattribute.Marshal(frontend)
if err != nil {
return err
}
fn(&dynamodb.ScanOutput{
Items: []map[string]*dynamodb.AttributeValue{
{
"id": &dynamodb.AttributeValue{
S: aws.String("test.traefik.io_backend"),
},
"name": &dynamodb.AttributeValue{
S: aws.String("test.traefik.io"),
},
"backend": attributeBackend,
},
},
}, false)
fn(&dynamodb.ScanOutput{
Items: []map[string]*dynamodb.AttributeValue{
{
"id": &dynamodb.AttributeValue{
S: aws.String("test.traefik.io_frontend"),
},
"name": &dynamodb.AttributeValue{
S: aws.String("test.traefik.io"),
},
"frontend": attributeFrontend,
},
},
}, true)
return nil
}
func TestLoadDynamoConfigSuccessful(t *testing.T) {
dbiface := &dynamoClient{
db: &mockDynamoDBCLient{
testWithError: false,
},
}
provider := DynamoDB{}
loadedConfig, err := provider.loadDynamoConfig(dbiface)
if err != nil {
t.Fatal(err)
}
expectedConfig := &types.Configuration{
Backends: map[string]*types.Backend{
"test.traefik.io": backend,
},
Frontends: map[string]*types.Frontend{
"test.traefik.io": frontend,
},
}
if !reflect.DeepEqual(loadedConfig, expectedConfig) {
t.Fatalf("Configurations did not match: %v %v", loadedConfig, expectedConfig)
}
}
func TestLoadDynamoConfigFailure(t *testing.T) {
dbiface := &dynamoClient{
db: &mockDynamoDBCLient{
testWithError: true,
},
}
provider := DynamoDB{}
_, err := provider.loadDynamoConfig(dbiface)
if err == nil {
t.Fatal("Expected error")
}
}
func TestCreateClientSuccessful(t *testing.T) {
provider := DynamoDB{
Region: "us-east-1",
}
_, err := provider.createClient()
if err != nil {
t.Fatal(err)
}
}
func TestCreateClientFailure(t *testing.T) {
provider := DynamoDB{}
_, err := provider.createClient()
if err == nil {
t.Fatal("Expected error")
}
}

View file

@ -380,6 +380,9 @@ func (server *Server) configureProviders() {
if server.globalConfiguration.Rancher != nil {
server.providers = append(server.providers, server.globalConfiguration.Rancher)
}
if server.globalConfiguration.DynamoDB != nil {
server.providers = append(server.providers, server.globalConfiguration.DynamoDB)
}
}
func (server *Server) startProviders() {

View file

@ -1019,6 +1019,59 @@ ExposedByDefault = false
# SecretKey = "XXXXXXXXXXX"
################################################################
# DynamoDB configuration backend
################################################################
# Enable DynamoDB configuration backend
#
# Optional
#
# [dynamodb]
# DynamoDB Table Name
#
# Optional
#
# TableName = "traefik"
# Enable watch DynamoDB changes
#
# Optional
#
# Watch = true
# Polling interval (in seconds)
#
# Optional
#
# RefreshSeconds = 15
# Region to use when connecting to AWS
#
# Required
#
# Region = "us-east-1"
# AccessKeyID to use when connecting to AWS
#
# Optional
#
# AccessKeyID = "abc"
# SecretAccessKey to use when connecting to AWS
#
# Optional
#
# SecretAccessKey = "123"
# Endpoint of dynamodb when testing locally
#
# Optional
#
# Endpoint = "http://localhost:8080"
################################################################
# Sample rules
################################################################

8077
vendor/github.com/aws/aws-sdk-go/service/dynamodb/api.go generated vendored Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,98 @@
package dynamodb
import (
"bytes"
"hash/crc32"
"io"
"io/ioutil"
"math"
"strconv"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/request"
)
type retryer struct {
client.DefaultRetryer
}
func (d retryer) RetryRules(r *request.Request) time.Duration {
delay := time.Duration(math.Pow(2, float64(r.RetryCount))) * 50
return delay * time.Millisecond
}
func init() {
initClient = func(c *client.Client) {
r := retryer{}
if c.Config.MaxRetries == nil || aws.IntValue(c.Config.MaxRetries) == aws.UseServiceDefaultRetries {
r.NumMaxRetries = 10
} else {
r.NumMaxRetries = *c.Config.MaxRetries
}
c.Retryer = r
c.Handlers.Build.PushBack(disableCompression)
c.Handlers.Unmarshal.PushFront(validateCRC32)
}
}
func drainBody(b io.ReadCloser, length int64) (out *bytes.Buffer, err error) {
if length < 0 {
length = 0
}
buf := bytes.NewBuffer(make([]byte, 0, length))
if _, err = buf.ReadFrom(b); err != nil {
return nil, err
}
if err = b.Close(); err != nil {
return nil, err
}
return buf, nil
}
func disableCompression(r *request.Request) {
r.HTTPRequest.Header.Set("Accept-Encoding", "identity")
}
func validateCRC32(r *request.Request) {
if r.Error != nil {
return // already have an error, no need to verify CRC
}
// Checksum validation is off, skip
if aws.BoolValue(r.Config.DisableComputeChecksums) {
return
}
// Try to get CRC from response
header := r.HTTPResponse.Header.Get("X-Amz-Crc32")
if header == "" {
return // No header, skip
}
expected, err := strconv.ParseUint(header, 10, 32)
if err != nil {
return // Could not determine CRC value, skip
}
buf, err := drainBody(r.HTTPResponse.Body, r.HTTPResponse.ContentLength)
if err != nil { // failed to read the response body, skip
return
}
// Reset body for subsequent reads
r.HTTPResponse.Body = ioutil.NopCloser(bytes.NewReader(buf.Bytes()))
// Compute the CRC checksum
crc := crc32.ChecksumIEEE(buf.Bytes())
if crc != uint32(expected) {
// CRC does not match, set a retryable error
r.Retryable = aws.Bool(true)
r.Error = awserr.New("CRC32CheckFailed", "CRC32 integrity check failed", nil)
}
}

View file

@ -0,0 +1,443 @@
package dynamodbattribute
import (
"bytes"
"encoding/json"
"fmt"
"reflect"
"runtime"
"strconv"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/dynamodb"
)
// ConvertToMap accepts a map[string]interface{} or struct and converts it to a
// map[string]*dynamodb.AttributeValue.
//
// If in contains any structs, it is first JSON encoded/decoded it to convert it
// to a map[string]interface{}, so `json` struct tags are respected.
//
// Deprecated: Use MarshalMap instead
func ConvertToMap(in interface{}) (item map[string]*dynamodb.AttributeValue, err error) {
defer func() {
if r := recover(); r != nil {
if e, ok := r.(runtime.Error); ok {
err = e
} else if s, ok := r.(string); ok {
err = fmt.Errorf(s)
} else {
err = r.(error)
}
item = nil
}
}()
if in == nil {
return nil, awserr.New("SerializationError",
"in must be a map[string]interface{} or struct, got <nil>", nil)
}
v := reflect.ValueOf(in)
if v.Kind() != reflect.Struct && !(v.Kind() == reflect.Map && v.Type().Key().Kind() == reflect.String) {
return nil, awserr.New("SerializationError",
fmt.Sprintf("in must be a map[string]interface{} or struct, got %s",
v.Type().String()),
nil)
}
if isTyped(reflect.TypeOf(in)) {
var out map[string]interface{}
in = convertToUntyped(in, out)
}
item = make(map[string]*dynamodb.AttributeValue)
for k, v := range in.(map[string]interface{}) {
item[k] = convertTo(v)
}
return item, nil
}
// ConvertFromMap accepts a map[string]*dynamodb.AttributeValue and converts it to a
// map[string]interface{} or struct.
//
// If v points to a struct, the result is first converted it to a
// map[string]interface{}, then JSON encoded/decoded it to convert to a struct,
// so `json` struct tags are respected.
//
// Deprecated: Use UnmarshalMap instead
func ConvertFromMap(item map[string]*dynamodb.AttributeValue, v interface{}) (err error) {
defer func() {
if r := recover(); r != nil {
if e, ok := r.(runtime.Error); ok {
err = e
} else if s, ok := r.(string); ok {
err = fmt.Errorf(s)
} else {
err = r.(error)
}
item = nil
}
}()
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
return awserr.New("SerializationError",
fmt.Sprintf("v must be a non-nil pointer to a map[string]interface{} or struct, got %s",
rv.Type()),
nil)
}
if rv.Elem().Kind() != reflect.Struct && !(rv.Elem().Kind() == reflect.Map && rv.Elem().Type().Key().Kind() == reflect.String) {
return awserr.New("SerializationError",
fmt.Sprintf("v must be a non-nil pointer to a map[string]interface{} or struct, got %s",
rv.Type()),
nil)
}
m := make(map[string]interface{})
for k, v := range item {
m[k] = convertFrom(v)
}
if isTyped(reflect.TypeOf(v)) {
err = convertToTyped(m, v)
} else {
rv.Elem().Set(reflect.ValueOf(m))
}
return err
}
// ConvertToList accepts an array or slice and converts it to a
// []*dynamodb.AttributeValue.
//
// Converting []byte fields to dynamodb.AttributeValue are only currently supported
// if the input is a map[string]interface{} type. []byte within typed structs are not
// converted correctly and are converted into base64 strings. This is a known bug,
// and will be fixed in a later release.
//
// If in contains any structs, it is first JSON encoded/decoded it to convert it
// to a []interface{}, so `json` struct tags are respected.
//
// Deprecated: Use MarshalList instead
func ConvertToList(in interface{}) (item []*dynamodb.AttributeValue, err error) {
defer func() {
if r := recover(); r != nil {
if e, ok := r.(runtime.Error); ok {
err = e
} else if s, ok := r.(string); ok {
err = fmt.Errorf(s)
} else {
err = r.(error)
}
item = nil
}
}()
if in == nil {
return nil, awserr.New("SerializationError",
"in must be an array or slice, got <nil>",
nil)
}
v := reflect.ValueOf(in)
if v.Kind() != reflect.Array && v.Kind() != reflect.Slice {
return nil, awserr.New("SerializationError",
fmt.Sprintf("in must be an array or slice, got %s",
v.Type().String()),
nil)
}
if isTyped(reflect.TypeOf(in)) {
var out []interface{}
in = convertToUntyped(in, out)
}
item = make([]*dynamodb.AttributeValue, 0, len(in.([]interface{})))
for _, v := range in.([]interface{}) {
item = append(item, convertTo(v))
}
return item, nil
}
// ConvertFromList accepts a []*dynamodb.AttributeValue and converts it to an array or
// slice.
//
// If v contains any structs, the result is first converted it to a
// []interface{}, then JSON encoded/decoded it to convert to a typed array or
// slice, so `json` struct tags are respected.
//
// Deprecated: Use UnmarshalList instead
func ConvertFromList(item []*dynamodb.AttributeValue, v interface{}) (err error) {
defer func() {
if r := recover(); r != nil {
if e, ok := r.(runtime.Error); ok {
err = e
} else if s, ok := r.(string); ok {
err = fmt.Errorf(s)
} else {
err = r.(error)
}
item = nil
}
}()
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
return awserr.New("SerializationError",
fmt.Sprintf("v must be a non-nil pointer to an array or slice, got %s",
rv.Type()),
nil)
}
if rv.Elem().Kind() != reflect.Array && rv.Elem().Kind() != reflect.Slice {
return awserr.New("SerializationError",
fmt.Sprintf("v must be a non-nil pointer to an array or slice, got %s",
rv.Type()),
nil)
}
l := make([]interface{}, 0, len(item))
for _, v := range item {
l = append(l, convertFrom(v))
}
if isTyped(reflect.TypeOf(v)) {
err = convertToTyped(l, v)
} else {
rv.Elem().Set(reflect.ValueOf(l))
}
return err
}
// ConvertTo accepts any interface{} and converts it to a *dynamodb.AttributeValue.
//
// If in contains any structs, it is first JSON encoded/decoded it to convert it
// to a interface{}, so `json` struct tags are respected.
//
// Deprecated: Use Marshal instead
func ConvertTo(in interface{}) (item *dynamodb.AttributeValue, err error) {
defer func() {
if r := recover(); r != nil {
if e, ok := r.(runtime.Error); ok {
err = e
} else if s, ok := r.(string); ok {
err = fmt.Errorf(s)
} else {
err = r.(error)
}
item = nil
}
}()
if in != nil && isTyped(reflect.TypeOf(in)) {
var out interface{}
in = convertToUntyped(in, out)
}
item = convertTo(in)
return item, nil
}
// ConvertFrom accepts a *dynamodb.AttributeValue and converts it to any interface{}.
//
// If v contains any structs, the result is first converted it to a interface{},
// then JSON encoded/decoded it to convert to a struct, so `json` struct tags
// are respected.
//
// Deprecated: Use Unmarshal instead
func ConvertFrom(item *dynamodb.AttributeValue, v interface{}) (err error) {
defer func() {
if r := recover(); r != nil {
if e, ok := r.(runtime.Error); ok {
err = e
} else if s, ok := r.(string); ok {
err = fmt.Errorf(s)
} else {
err = r.(error)
}
item = nil
}
}()
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
return awserr.New("SerializationError",
fmt.Sprintf("v must be a non-nil pointer to an interface{} or struct, got %s",
rv.Type()),
nil)
}
if rv.Elem().Kind() != reflect.Interface && rv.Elem().Kind() != reflect.Struct {
return awserr.New("SerializationError",
fmt.Sprintf("v must be a non-nil pointer to an interface{} or struct, got %s",
rv.Type()),
nil)
}
res := convertFrom(item)
if isTyped(reflect.TypeOf(v)) {
err = convertToTyped(res, v)
} else if res != nil {
rv.Elem().Set(reflect.ValueOf(res))
}
return err
}
func isTyped(v reflect.Type) bool {
switch v.Kind() {
case reflect.Struct:
return true
case reflect.Array, reflect.Slice:
if isTyped(v.Elem()) {
return true
}
case reflect.Map:
if isTyped(v.Key()) {
return true
}
if isTyped(v.Elem()) {
return true
}
case reflect.Ptr:
return isTyped(v.Elem())
}
return false
}
func convertToUntyped(in, out interface{}) interface{} {
b, err := json.Marshal(in)
if err != nil {
panic(err)
}
decoder := json.NewDecoder(bytes.NewReader(b))
decoder.UseNumber()
err = decoder.Decode(&out)
if err != nil {
panic(err)
}
return out
}
func convertToTyped(in, out interface{}) error {
b, err := json.Marshal(in)
if err != nil {
return err
}
decoder := json.NewDecoder(bytes.NewReader(b))
return decoder.Decode(&out)
}
func convertTo(in interface{}) *dynamodb.AttributeValue {
a := &dynamodb.AttributeValue{}
if in == nil {
a.NULL = new(bool)
*a.NULL = true
return a
}
if m, ok := in.(map[string]interface{}); ok {
a.M = make(map[string]*dynamodb.AttributeValue)
for k, v := range m {
a.M[k] = convertTo(v)
}
return a
}
v := reflect.ValueOf(in)
switch v.Kind() {
case reflect.Bool:
a.BOOL = new(bool)
*a.BOOL = v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
a.N = new(string)
*a.N = strconv.FormatInt(v.Int(), 10)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
a.N = new(string)
*a.N = strconv.FormatUint(v.Uint(), 10)
case reflect.Float32, reflect.Float64:
a.N = new(string)
*a.N = strconv.FormatFloat(v.Float(), 'f', -1, 64)
case reflect.String:
if n, ok := in.(json.Number); ok {
a.N = new(string)
*a.N = n.String()
} else {
a.S = new(string)
*a.S = v.String()
}
case reflect.Slice:
switch v.Type() {
case reflect.TypeOf(([]byte)(nil)):
a.B = v.Bytes()
default:
a.L = make([]*dynamodb.AttributeValue, v.Len())
for i := 0; i < v.Len(); i++ {
a.L[i] = convertTo(v.Index(i).Interface())
}
}
default:
panic(fmt.Sprintf("the type %s is not supported", v.Type().String()))
}
return a
}
func convertFrom(a *dynamodb.AttributeValue) interface{} {
if a.S != nil {
return *a.S
}
if a.N != nil {
// Number is tricky b/c we don't know which numeric type to use. Here we
// simply try the different types from most to least restrictive.
if n, err := strconv.ParseInt(*a.N, 10, 64); err == nil {
return int(n)
}
if n, err := strconv.ParseUint(*a.N, 10, 64); err == nil {
return uint(n)
}
n, err := strconv.ParseFloat(*a.N, 64)
if err != nil {
panic(err)
}
return n
}
if a.BOOL != nil {
return *a.BOOL
}
if a.NULL != nil {
return nil
}
if a.M != nil {
m := make(map[string]interface{})
for k, v := range a.M {
m[k] = convertFrom(v)
}
return m
}
if a.L != nil {
l := make([]interface{}, len(a.L))
for index, v := range a.L {
l[index] = convertFrom(v)
}
return l
}
if a.B != nil {
return a.B
}
panic(fmt.Sprintf("%#v is not a supported dynamodb.AttributeValue", a))
}

View file

@ -0,0 +1,680 @@
package dynamodbattribute
import (
"fmt"
"reflect"
"strconv"
"time"
"github.com/aws/aws-sdk-go/service/dynamodb"
)
// An Unmarshaler is an interface to provide custom unmarshaling of
// AttributeValues. Use this to provide custom logic determining
// how AttributeValues should be unmarshaled.
// type ExampleUnmarshaler struct {
// Value int
// }
//
// type (u *exampleUnmarshaler) UnmarshalDynamoDBAttributeValue(av *dynamodb.AttributeValue) error {
// if av.N == nil {
// return nil
// }
//
// n, err := strconv.ParseInt(*av.N, 10, 0)
// if err != nil {
// return err
// }
//
// u.Value = n
// return nil
// }
type Unmarshaler interface {
UnmarshalDynamoDBAttributeValue(*dynamodb.AttributeValue) error
}
// Unmarshal will unmarshal DynamoDB AttributeValues to Go value types.
// Both generic interface{} and concrete types are valid unmarshal
// destination types.
//
// Unmarshal will allocate maps, slices, and pointers as needed to
// unmarshal the AttributeValue into the provided type value.
//
// When unmarshaling AttributeValues into structs Unmarshal matches
// the field names of the struct to the AttributeValue Map keys.
// Initially it will look for exact field name matching, but will
// fall back to case insensitive if not exact match is found.
//
// With the exception of omitempty, omitemptyelem, binaryset, numberset
// and stringset all struct tags used by Marshal are also used by
// Unmarshal.
//
// When decoding AttributeValues to interfaces Unmarshal will use the
// following types.
//
// []byte, AV Binary (B)
// [][]byte, AV Binary Set (BS)
// bool, AV Boolean (BOOL)
// []interface{}, AV List (L)
// map[string]interface{}, AV Map (M)
// float64, AV Number (N)
// Number, AV Number (N) with UseNumber set
// []float64, AV Number Set (NS)
// []Number, AV Number Set (NS) with UseNumber set
// string, AV String (S)
// []string, AV String Set (SS)
//
// If the Decoder option, UseNumber is set numbers will be unmarshaled
// as Number values instead of float64. Use this to maintain the original
// string formating of the number as it was represented in the AttributeValue.
// In addition provides additional opportunities to parse the number
// string based on individual use cases.
//
// When unmarshaling any error that occurs will halt the unmarshal
// and return the error.
//
// The output value provided must be a non-nil pointer
func Unmarshal(av *dynamodb.AttributeValue, out interface{}) error {
return NewDecoder().Decode(av, out)
}
// UnmarshalMap is an alias for Unmarshal which unmarshals from
// a map of AttributeValues.
//
// The output value provided must be a non-nil pointer
func UnmarshalMap(m map[string]*dynamodb.AttributeValue, out interface{}) error {
return NewDecoder().Decode(&dynamodb.AttributeValue{M: m}, out)
}
// UnmarshalList is an alias for Unmarshal func which unmarshals
// a slice of AttributeValues.
//
// The output value provided must be a non-nil pointer
func UnmarshalList(l []*dynamodb.AttributeValue, out interface{}) error {
return NewDecoder().Decode(&dynamodb.AttributeValue{L: l}, out)
}
// UnmarshalListOfMaps is an alias for Unmarshal func which unmarshals a
// slice of maps of attribute values.
//
// This is useful for when you need to unmarshal the Items from a DynamoDB
// Query API call.
//
// The output value provided must be a non-nil pointer
func UnmarshalListOfMaps(l []map[string]*dynamodb.AttributeValue, out interface{}) error {
items := make([]*dynamodb.AttributeValue, len(l))
for i, m := range l {
items[i] = &dynamodb.AttributeValue{M: m}
}
return UnmarshalList(items, out)
}
// A Decoder provides unmarshaling AttributeValues to Go value types.
type Decoder struct {
MarshalOptions
// Instructs the decoder to decode AttributeValue Numbers as
// Number type instead of float64 when the destination type
// is interface{}. Similar to encoding/json.Number
UseNumber bool
}
// NewDecoder creates a new Decoder with default configuration. Use
// the `opts` functional options to override the default configuration.
func NewDecoder(opts ...func(*Decoder)) *Decoder {
d := &Decoder{
MarshalOptions: MarshalOptions{
SupportJSONTags: true,
},
}
for _, o := range opts {
o(d)
}
return d
}
// Decode will unmarshal an AttributeValue into a Go value type. An error
// will be return if the decoder is unable to unmarshal the AttributeValue
// to the provide Go value type.
//
// The output value provided must be a non-nil pointer
func (d *Decoder) Decode(av *dynamodb.AttributeValue, out interface{}, opts ...func(*Decoder)) error {
v := reflect.ValueOf(out)
if v.Kind() != reflect.Ptr || v.IsNil() || !v.IsValid() {
return &InvalidUnmarshalError{Type: reflect.TypeOf(out)}
}
return d.decode(av, v, tag{})
}
var stringInterfaceMapType = reflect.TypeOf(map[string]interface{}(nil))
var byteSliceType = reflect.TypeOf([]byte(nil))
var byteSliceSlicetype = reflect.TypeOf([][]byte(nil))
var numberType = reflect.TypeOf(Number(""))
func (d *Decoder) decode(av *dynamodb.AttributeValue, v reflect.Value, fieldTag tag) error {
var u Unmarshaler
if av == nil || av.NULL != nil {
u, v = indirect(v, true)
if u != nil {
return u.UnmarshalDynamoDBAttributeValue(av)
}
return d.decodeNull(v)
}
u, v = indirect(v, false)
if u != nil {
return u.UnmarshalDynamoDBAttributeValue(av)
}
switch {
case len(av.B) != 0:
return d.decodeBinary(av.B, v)
case av.BOOL != nil:
return d.decodeBool(av.BOOL, v)
case len(av.BS) != 0:
return d.decodeBinarySet(av.BS, v)
case len(av.L) != 0:
return d.decodeList(av.L, v)
case len(av.M) != 0:
return d.decodeMap(av.M, v)
case av.N != nil:
return d.decodeNumber(av.N, v)
case len(av.NS) != 0:
return d.decodeNumberSet(av.NS, v)
case av.S != nil:
return d.decodeString(av.S, v, fieldTag)
case len(av.SS) != 0:
return d.decodeStringSet(av.SS, v)
}
return nil
}
func (d *Decoder) decodeBinary(b []byte, v reflect.Value) error {
if v.Kind() == reflect.Interface {
buf := make([]byte, len(b))
copy(buf, b)
v.Set(reflect.ValueOf(buf))
return nil
}
if v.Kind() != reflect.Slice {
return &UnmarshalTypeError{Value: "binary", Type: v.Type()}
}
if v.Type() == byteSliceType {
// Optimization for []byte types
if v.IsNil() || v.Cap() < len(b) {
v.Set(reflect.MakeSlice(byteSliceType, len(b), len(b)))
} else if v.Len() != len(b) {
v.SetLen(len(b))
}
copy(v.Interface().([]byte), b)
return nil
}
switch v.Type().Elem().Kind() {
case reflect.Uint8:
// Fallback to reflection copy for type aliased of []byte type
if v.IsNil() || v.Cap() < len(b) {
v.Set(reflect.MakeSlice(v.Type(), len(b), len(b)))
} else if v.Len() != len(b) {
v.SetLen(len(b))
}
for i := 0; i < len(b); i++ {
v.Index(i).SetUint(uint64(b[i]))
}
default:
if v.Kind() == reflect.Array && v.Type().Elem().Kind() == reflect.Uint8 {
reflect.Copy(v, reflect.ValueOf(b))
break
}
return &UnmarshalTypeError{Value: "binary", Type: v.Type()}
}
return nil
}
func (d *Decoder) decodeBool(b *bool, v reflect.Value) error {
switch v.Kind() {
case reflect.Bool, reflect.Interface:
v.Set(reflect.ValueOf(*b).Convert(v.Type()))
default:
return &UnmarshalTypeError{Value: "bool", Type: v.Type()}
}
return nil
}
func (d *Decoder) decodeBinarySet(bs [][]byte, v reflect.Value) error {
switch v.Kind() {
case reflect.Slice:
// Make room for the slice elements if needed
if v.IsNil() || v.Cap() < len(bs) {
// What about if ignoring nil/empty values?
v.Set(reflect.MakeSlice(v.Type(), 0, len(bs)))
}
case reflect.Array:
// Limited to capacity of existing array.
case reflect.Interface:
set := make([][]byte, len(bs))
for i, b := range bs {
if err := d.decodeBinary(b, reflect.ValueOf(&set[i]).Elem()); err != nil {
return err
}
}
v.Set(reflect.ValueOf(set))
return nil
default:
return &UnmarshalTypeError{Value: "binary set", Type: v.Type()}
}
for i := 0; i < v.Cap() && i < len(bs); i++ {
v.SetLen(i + 1)
u, elem := indirect(v.Index(i), false)
if u != nil {
return u.UnmarshalDynamoDBAttributeValue(&dynamodb.AttributeValue{BS: bs})
}
if err := d.decodeBinary(bs[i], elem); err != nil {
return err
}
}
return nil
}
func (d *Decoder) decodeNumber(n *string, v reflect.Value) error {
switch v.Kind() {
case reflect.Interface:
i, err := d.decodeNumberToInterface(n)
if err != nil {
return err
}
v.Set(reflect.ValueOf(i))
return nil
case reflect.String:
if v.Type() == numberType { // Support Number value type
v.Set(reflect.ValueOf(Number(*n)))
return nil
}
v.Set(reflect.ValueOf(*n))
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
i, err := strconv.ParseInt(*n, 10, 64)
if err != nil {
return err
}
if v.OverflowInt(i) {
return &UnmarshalTypeError{
Value: fmt.Sprintf("number overflow, %s", *n),
Type: v.Type(),
}
}
v.SetInt(i)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
i, err := strconv.ParseUint(*n, 10, 64)
if err != nil {
return err
}
if v.OverflowUint(i) {
return &UnmarshalTypeError{
Value: fmt.Sprintf("number overflow, %s", *n),
Type: v.Type(),
}
}
v.SetUint(i)
case reflect.Float32, reflect.Float64:
i, err := strconv.ParseFloat(*n, 64)
if err != nil {
return err
}
if v.OverflowFloat(i) {
return &UnmarshalTypeError{
Value: fmt.Sprintf("number overflow, %s", *n),
Type: v.Type(),
}
}
v.SetFloat(i)
default:
return &UnmarshalTypeError{Value: "number", Type: v.Type()}
}
return nil
}
func (d *Decoder) decodeNumberToInterface(n *string) (interface{}, error) {
if d.UseNumber {
return Number(*n), nil
}
// Default to float64 for all numbers
return strconv.ParseFloat(*n, 64)
}
func (d *Decoder) decodeNumberSet(ns []*string, v reflect.Value) error {
switch v.Kind() {
case reflect.Slice:
// Make room for the slice elements if needed
if v.IsNil() || v.Cap() < len(ns) {
// What about if ignoring nil/empty values?
v.Set(reflect.MakeSlice(v.Type(), 0, len(ns)))
}
case reflect.Array:
// Limited to capacity of existing array.
case reflect.Interface:
if d.UseNumber {
set := make([]Number, len(ns))
for i, n := range ns {
if err := d.decodeNumber(n, reflect.ValueOf(&set[i]).Elem()); err != nil {
return err
}
}
v.Set(reflect.ValueOf(set))
} else {
set := make([]float64, len(ns))
for i, n := range ns {
if err := d.decodeNumber(n, reflect.ValueOf(&set[i]).Elem()); err != nil {
return err
}
}
v.Set(reflect.ValueOf(set))
}
return nil
default:
return &UnmarshalTypeError{Value: "number set", Type: v.Type()}
}
for i := 0; i < v.Cap() && i < len(ns); i++ {
v.SetLen(i + 1)
u, elem := indirect(v.Index(i), false)
if u != nil {
return u.UnmarshalDynamoDBAttributeValue(&dynamodb.AttributeValue{NS: ns})
}
if err := d.decodeNumber(ns[i], elem); err != nil {
return err
}
}
return nil
}
func (d *Decoder) decodeList(avList []*dynamodb.AttributeValue, v reflect.Value) error {
switch v.Kind() {
case reflect.Slice:
// Make room for the slice elements if needed
if v.IsNil() || v.Cap() < len(avList) {
// What about if ignoring nil/empty values?
v.Set(reflect.MakeSlice(v.Type(), 0, len(avList)))
}
case reflect.Array:
// Limited to capacity of existing array.
case reflect.Interface:
s := make([]interface{}, len(avList))
for i, av := range avList {
if err := d.decode(av, reflect.ValueOf(&s[i]).Elem(), tag{}); err != nil {
return err
}
}
v.Set(reflect.ValueOf(s))
return nil
default:
return &UnmarshalTypeError{Value: "list", Type: v.Type()}
}
// If v is not a slice, array
for i := 0; i < v.Cap() && i < len(avList); i++ {
v.SetLen(i + 1)
if err := d.decode(avList[i], v.Index(i), tag{}); err != nil {
return err
}
}
return nil
}
func (d *Decoder) decodeMap(avMap map[string]*dynamodb.AttributeValue, v reflect.Value) error {
switch v.Kind() {
case reflect.Map:
t := v.Type()
if t.Key().Kind() != reflect.String {
return &UnmarshalTypeError{Value: "map string key", Type: t.Key()}
}
if v.IsNil() {
v.Set(reflect.MakeMap(t))
}
case reflect.Struct:
case reflect.Interface:
v.Set(reflect.MakeMap(stringInterfaceMapType))
v = v.Elem()
default:
return &UnmarshalTypeError{Value: "map", Type: v.Type()}
}
if v.Kind() == reflect.Map {
for k, av := range avMap {
key := reflect.ValueOf(k)
elem := reflect.New(v.Type().Elem()).Elem()
if err := d.decode(av, elem, tag{}); err != nil {
return err
}
v.SetMapIndex(key, elem)
}
} else if v.Kind() == reflect.Struct {
fields := unionStructFields(v.Type(), d.MarshalOptions)
for k, av := range avMap {
if f, ok := fieldByName(fields, k); ok {
fv := fieldByIndex(v, f.Index, func(v *reflect.Value) bool {
v.Set(reflect.New(v.Type().Elem()))
return true // to continue the loop.
})
if err := d.decode(av, fv, f.tag); err != nil {
return err
}
}
}
}
return nil
}
func (d *Decoder) decodeNull(v reflect.Value) error {
if v.IsValid() && v.CanSet() {
v.Set(reflect.Zero(v.Type()))
}
return nil
}
func (d *Decoder) decodeString(s *string, v reflect.Value, fieldTag tag) error {
if fieldTag.AsString {
return d.decodeNumber(s, v)
}
// To maintain backwards compatibility with ConvertFrom family of methods which
// converted strings to time.Time structs
if _, ok := v.Interface().(time.Time); ok {
t, err := time.Parse(time.RFC3339, *s)
if err != nil {
return err
}
v.Set(reflect.ValueOf(t))
return nil
}
switch v.Kind() {
case reflect.String:
v.SetString(*s)
case reflect.Interface:
// Ensure type aliasing is handled properly
v.Set(reflect.ValueOf(*s).Convert(v.Type()))
default:
return &UnmarshalTypeError{Value: "string", Type: v.Type()}
}
return nil
}
func (d *Decoder) decodeStringSet(ss []*string, v reflect.Value) error {
switch v.Kind() {
case reflect.Slice:
// Make room for the slice elements if needed
if v.IsNil() || v.Cap() < len(ss) {
v.Set(reflect.MakeSlice(v.Type(), 0, len(ss)))
}
case reflect.Array:
// Limited to capacity of existing array.
case reflect.Interface:
set := make([]string, len(ss))
for i, s := range ss {
if err := d.decodeString(s, reflect.ValueOf(&set[i]).Elem(), tag{}); err != nil {
return err
}
}
v.Set(reflect.ValueOf(set))
return nil
default:
return &UnmarshalTypeError{Value: "string set", Type: v.Type()}
}
for i := 0; i < v.Cap() && i < len(ss); i++ {
v.SetLen(i + 1)
u, elem := indirect(v.Index(i), false)
if u != nil {
return u.UnmarshalDynamoDBAttributeValue(&dynamodb.AttributeValue{SS: ss})
}
if err := d.decodeString(ss[i], elem, tag{}); err != nil {
return err
}
}
return nil
}
// indirect will walk a value's interface or pointer value types. Returning
// the final value or the value a unmarshaler is defined on.
//
// Based on the enoding/json type reflect value type indirection in Go Stdlib
// https://golang.org/src/encoding/json/decode.go indirect func.
func indirect(v reflect.Value, decodingNull bool) (Unmarshaler, reflect.Value) {
if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
v = v.Addr()
}
for {
if v.Kind() == reflect.Interface && !v.IsNil() {
e := v.Elem()
if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) {
v = e
continue
}
}
if v.Kind() != reflect.Ptr {
break
}
if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() {
break
}
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
if v.Type().NumMethod() > 0 {
if u, ok := v.Interface().(Unmarshaler); ok {
return u, reflect.Value{}
}
}
v = v.Elem()
}
return nil, v
}
// A Number represents a Attributevalue number literal.
type Number string
// Float64 attempts to cast the number ot a float64, returning
// the result of the case or error if the case failed.
func (n Number) Float64() (float64, error) {
return strconv.ParseFloat(string(n), 64)
}
// Int64 attempts to cast the number ot a int64, returning
// the result of the case or error if the case failed.
func (n Number) Int64() (int64, error) {
return strconv.ParseInt(string(n), 10, 64)
}
// Uint64 attempts to cast the number ot a uint64, returning
// the result of the case or error if the case failed.
func (n Number) Uint64() (uint64, error) {
return strconv.ParseUint(string(n), 10, 64)
}
// String returns the raw number represented as a string
func (n Number) String() string {
return string(n)
}
type emptyOrigError struct{}
func (e emptyOrigError) OrigErr() error {
return nil
}
// An UnmarshalTypeError is an error type representing a error
// unmarshaling the AttributeValue's element to a Go value type.
// Includes details about the AttributeValue type and Go value type.
type UnmarshalTypeError struct {
emptyOrigError
Value string
Type reflect.Type
}
// Error returns the string representation of the error.
// satisfying the error interface
func (e *UnmarshalTypeError) Error() string {
return fmt.Sprintf("%s: %s", e.Code(), e.Message())
}
// Code returns the code of the error, satisfying the awserr.Error
// interface.
func (e *UnmarshalTypeError) Code() string {
return "UnmarshalTypeError"
}
// Message returns the detailed message of the error, satisfying
// the awserr.Error interface.
func (e *UnmarshalTypeError) Message() string {
return "cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String()
}
// An InvalidUnmarshalError is an error type representing an invalid type
// encountered while unmarshaling a AttributeValue to a Go value type.
type InvalidUnmarshalError struct {
emptyOrigError
Type reflect.Type
}
// Error returns the string representation of the error.
// satisfying the error interface
func (e *InvalidUnmarshalError) Error() string {
return fmt.Sprintf("%s: %s", e.Code(), e.Message())
}
// Code returns the code of the error, satisfying the awserr.Error
// interface.
func (e *InvalidUnmarshalError) Code() string {
return "InvalidUnmarshalError"
}
// Message returns the detailed message of the error, satisfying
// the awserr.Error interface.
func (e *InvalidUnmarshalError) Message() string {
if e.Type == nil {
return "cannot unmarshal to nil value"
}
if e.Type.Kind() != reflect.Ptr {
return "cannot unmarshal to non-pointer value, got " + e.Type.String()
}
return "cannot unmarshal to nil value, " + e.Type.String()
}

View file

@ -0,0 +1,67 @@
// Package dynamodbattribute provides marshaling utilities for marshaling to
// dynamodb.AttributeValue types and unmarshaling to Go value types. These
// utilities allow you to marshal slices, maps, structs, and scalar values
// to and from dynamodb.AttributeValue. These are useful when marshaling
// Go value tyes to dynamodb.AttributeValue for DynamoDB requests, or
// unmarshaling the dynamodb.AttributeValue back into a Go value type.
//
// Marshal Go value types to dynamodb.AttributeValue: See (ExampleMarshal)
//
// type Record struct {
// MyField string
// Letters []string
// A2Num map[string]int
// }
//
// ...
//
// r := Record{
// MyField: "dynamodbattribute.Marshal example",
// Letters: []string{"a", "b", "c", "d"},
// A2Num: map[string]int{"a": 1, "b": 2, "c": 3},
// }
// av, err := dynamodbattribute.Marshal(r)
// fmt.Println(av, err)
//
// Unmarshal dynamodb.AttributeValue to Go value type: See (ExampleUnmarshal)
//
// r2 := Record{}
// err = dynamodbattribute.Unmarshal(av, &r2)
// fmt.Println(err, reflect.DeepEqual(r, r2))
//
// Marshal Go value type for DynamoDB.PutItem:
//
// sess, err := session.NewSession()
// if err != nil {
// fmt.Println("Failed create session", err)
// return
// }
//
// svc := dynamodb.New(sess)
// item, err := dynamodbattribute.MarshalMap(r)
// if err != nil {
// fmt.Println("Failed to convert", err)
// return
// }
// result, err := svc.PutItem(&dynamodb.PutItemInput{
// Item: item,
// TableName: aws.String("exampleTable"),
// })
//
//
//
// The ConvertTo, ConvertToList, ConvertToMap, ConvertFrom, ConvertFromMap
// and ConvertFromList methods have been deprecated. The Marshal and Unmarshal
// functions should be used instead. The ConvertTo|From marshallers do not
// support BinarySet, NumberSet, nor StringSets, and will incorrect marshal
// binary data fields in structs as base64 strings.
//
// The Marshal and Unmarshal functions correct this behavior, and removes
// the reliance on encoding.json. `json` struct tags are still supported. In
// addition support for a new struct tag `dynamodbav` was added. Support for
// the json.Marshaler and json.Unmarshaler interfaces have been removed and
// replaced with have been replaced with dynamodbattribute.Marshaler and
// dynamodbattribute.Unmarshaler interfaces.
//
// `time.Time` is marshaled as RFC3339 format.
package dynamodbattribute

View file

@ -0,0 +1,585 @@
package dynamodbattribute
import (
"fmt"
"reflect"
"strconv"
"time"
"github.com/aws/aws-sdk-go/service/dynamodb"
)
// A Marshaler is an interface to provide custom marshaling of Go value types
// to AttributeValues. Use this to provide custom logic determining how a
// Go Value type should be marshaled.
//
// type ExampleMarshaler struct {
// Value int
// }
// type (m *ExampleMarshaler) MarshalDynamoDBAttributeValue(av *dynamodb.AttributeValue) error {
// n := fmt.Sprintf("%v", m.Value)
// av.N = &n
//
// return nil
// }
//
type Marshaler interface {
MarshalDynamoDBAttributeValue(*dynamodb.AttributeValue) error
}
// Marshal will serialize the passed in Go value type into a DynamoDB AttributeValue
// type. This value can be used in DynamoDB API operations to simplify marshaling
// your Go value types into AttributeValues.
//
// Marshal will recursively transverse the passed in value marshaling its
// contents into a AttributeValue. Marshal supports basic scalars
// (int,uint,float,bool,string), maps, slices, and structs. Anonymous
// nested types are flattened based on Go anonymous type visibility.
//
// Marshaling slices to AttributeValue will default to a List for all
// types except for []byte and [][]byte. []byte will be marshaled as
// Binary data (B), and [][]byte will be marshaled as binary data set
// (BS).
//
// `dynamodbav` struct tag can be used to control how the value will be
// marshaled into a AttributeValue.
//
// // Field is ignored
// Field int `dynamodbav:"-"`
//
// // Field AttributeValue map key "myName"
// Field int `dynamodbav:"myName"`
//
// // Field AttributeValue map key "myName", and
// // Field is omitted if it is empty
// Field int `dynamodbav:"myName,omitempty"`
//
// // Field AttributeValue map key "Field", and
// // Field is omitted if it is empty
// Field int `dynamodbav:",omitempty"`
//
// // Field's elems will be omitted if empty
// // only valid for slices, and maps.
// Field []string `dynamodbav:",omitemptyelem"`
//
// // Field will be marshaled as a AttributeValue string
// // only value for number types, (int,uint,float)
// Field int `dynamodbav:",string"`
//
// // Field will be marshaled as a binary set
// Field [][]byte `dynamodbav:",binaryset"`
//
// // Field will be marshaled as a number set
// Field []int `dynamodbav:",numberset"`
//
// // Field will be marshaled as a string set
// Field []string `dynamodbav:",stringset"`
//
// The omitempty tag is only used during Marshaling and is ignored for
// Unmarshal. Any zero value or a value when marshaled results in a
// AttributeValue NULL will be added to AttributeValue Maps during struct
// marshal. The omitemptyelem tag works the same as omitempty except it
// applies to maps and slices instead of struct fields, and will not be
// included in the marshaled AttributeValue Map, List, or Set.
//
// For convenience and backwards compatibility with ConvertTo functions
// json struct tags are supported by the Marshal and Unmarshal. If
// both json and dynamodbav struct tags are provided the json tag will
// be ignored in favor of dynamodbav.
//
// All struct fields and with anonymous fields, are marshaled unless the
// any of the following conditions are meet.
//
// - the field is not exported
// - json or dynamodbav field tag is "-"
// - json or dynamodbav field tag specifies "omitempty", and is empty.
//
// Pointer and interfaces values encode as the value pointed to or contained
// in the interface. A nil value encodes as the AttributeValue NULL value.
//
// Channel, complex, and function values are not encoded and will be skipped
// when walking the value to be marshaled.
//
// When marshaling any error that occurs will halt the marshal and return
// the error.
//
// Marshal cannot represent cyclic data structures and will not handle them.
// Passing cyclic structures to Marshal will result in an infinite recursion.
func Marshal(in interface{}) (*dynamodb.AttributeValue, error) {
return NewEncoder().Encode(in)
}
// MarshalMap is an alias for Marshal func which marshals Go value
// type to a map of AttributeValues.
func MarshalMap(in interface{}) (map[string]*dynamodb.AttributeValue, error) {
av, err := NewEncoder().Encode(in)
if err != nil || av == nil || av.M == nil {
return map[string]*dynamodb.AttributeValue{}, err
}
return av.M, nil
}
// MarshalList is an alias for Marshal func which marshals Go value
// type to a slice of AttributeValues.
func MarshalList(in interface{}) ([]*dynamodb.AttributeValue, error) {
av, err := NewEncoder().Encode(in)
if err != nil || av == nil || av.L == nil {
return []*dynamodb.AttributeValue{}, err
}
return av.L, nil
}
// A MarshalOptions is a collection of options shared between marshaling
// and unmarshaling
type MarshalOptions struct {
// States that the encoding/json struct tags should be supported.
// if a `dynamodbav` struct tag is also provided the encoding/json
// tag will be ignored.
//
// Enabled by default.
SupportJSONTags bool
}
// An Encoder provides marshaling Go value types to AttributeValues.
type Encoder struct {
MarshalOptions
// Empty strings, "", will be marked as NULL AttributeValue types.
// Empty strings are not valid values for DynamoDB. Will not apply
// to lists, sets, or maps. Use the struct tag `omitemptyelem`
// to skip empty (zero) values in lists, sets and maps.
//
// Enabled by default.
NullEmptyString bool
}
// NewEncoder creates a new Encoder with default configuration. Use
// the `opts` functional options to override the default configuration.
func NewEncoder(opts ...func(*Encoder)) *Encoder {
e := &Encoder{
MarshalOptions: MarshalOptions{
SupportJSONTags: true,
},
NullEmptyString: true,
}
for _, o := range opts {
o(e)
}
return e
}
// Encode will marshal a Go value type to an AttributeValue. Returning
// the AttributeValue constructed or error.
func (e *Encoder) Encode(in interface{}) (*dynamodb.AttributeValue, error) {
av := &dynamodb.AttributeValue{}
if err := e.encode(av, reflect.ValueOf(in), tag{}); err != nil {
return nil, err
}
return av, nil
}
func fieldByIndex(v reflect.Value, index []int,
OnEmbeddedNilStruct func(*reflect.Value) bool) reflect.Value {
fv := v
for i, x := range index {
if i > 0 {
if fv.Kind() == reflect.Ptr && fv.Type().Elem().Kind() == reflect.Struct {
if fv.IsNil() && !OnEmbeddedNilStruct(&fv) {
break
}
fv = fv.Elem()
}
}
fv = fv.Field(x)
}
return fv
}
func (e *Encoder) encode(av *dynamodb.AttributeValue, v reflect.Value, fieldTag tag) error {
// We should check for omitted values first before dereferencing.
if fieldTag.OmitEmpty && emptyValue(v) {
encodeNull(av)
return nil
}
// Handle both pointers and interface conversion into types
v = valueElem(v)
if v.Kind() != reflect.Invalid {
if used, err := tryMarshaler(av, v); used {
return err
}
}
switch v.Kind() {
case reflect.Invalid:
encodeNull(av)
case reflect.Struct:
return e.encodeStruct(av, v)
case reflect.Map:
return e.encodeMap(av, v, fieldTag)
case reflect.Slice, reflect.Array:
return e.encodeSlice(av, v, fieldTag)
case reflect.Chan, reflect.Func, reflect.UnsafePointer:
// do nothing for unsupported types
default:
return e.encodeScalar(av, v, fieldTag)
}
return nil
}
func (e *Encoder) encodeStruct(av *dynamodb.AttributeValue, v reflect.Value) error {
// To maintain backwards compatibility with ConvertTo family of methods which
// converted time.Time structs to strings
if t, ok := v.Interface().(time.Time); ok {
s := t.Format(time.RFC3339Nano)
av.S = &s
return nil
}
av.M = map[string]*dynamodb.AttributeValue{}
fields := unionStructFields(v.Type(), e.MarshalOptions)
for _, f := range fields {
if f.Name == "" {
return &InvalidMarshalError{msg: "map key cannot be empty"}
}
found := true
fv := fieldByIndex(v, f.Index, func(v *reflect.Value) bool {
found = false
return false // to break the loop.
})
if !found {
continue
}
elem := &dynamodb.AttributeValue{}
err := e.encode(elem, fv, f.tag)
if err != nil {
return err
}
skip, err := keepOrOmitEmpty(f.OmitEmpty, elem, err)
if err != nil {
return err
} else if skip {
continue
}
av.M[f.Name] = elem
}
if len(av.M) == 0 {
encodeNull(av)
}
return nil
}
func (e *Encoder) encodeMap(av *dynamodb.AttributeValue, v reflect.Value, fieldTag tag) error {
av.M = map[string]*dynamodb.AttributeValue{}
for _, key := range v.MapKeys() {
keyName := fmt.Sprint(key.Interface())
if keyName == "" {
return &InvalidMarshalError{msg: "map key cannot be empty"}
}
elemVal := v.MapIndex(key)
elem := &dynamodb.AttributeValue{}
err := e.encode(elem, elemVal, tag{})
skip, err := keepOrOmitEmpty(fieldTag.OmitEmptyElem, elem, err)
if err != nil {
return err
} else if skip {
continue
}
av.M[keyName] = elem
}
if len(av.M) == 0 {
encodeNull(av)
}
return nil
}
func (e *Encoder) encodeSlice(av *dynamodb.AttributeValue, v reflect.Value, fieldTag tag) error {
switch v.Type().Elem().Kind() {
case reflect.Uint8:
b := v.Bytes()
if len(b) == 0 {
encodeNull(av)
return nil
}
av.B = append([]byte{}, b...)
default:
var elemFn func(dynamodb.AttributeValue) error
if fieldTag.AsBinSet || v.Type() == byteSliceSlicetype { // Binary Set
av.BS = make([][]byte, 0, v.Len())
elemFn = func(elem dynamodb.AttributeValue) error {
if elem.B == nil {
return &InvalidMarshalError{msg: "binary set must only contain non-nil byte slices"}
}
av.BS = append(av.BS, elem.B)
return nil
}
} else if fieldTag.AsNumSet { // Number Set
av.NS = make([]*string, 0, v.Len())
elemFn = func(elem dynamodb.AttributeValue) error {
if elem.N == nil {
return &InvalidMarshalError{msg: "number set must only contain non-nil string numbers"}
}
av.NS = append(av.NS, elem.N)
return nil
}
} else if fieldTag.AsStrSet { // String Set
av.SS = make([]*string, 0, v.Len())
elemFn = func(elem dynamodb.AttributeValue) error {
if elem.S == nil {
return &InvalidMarshalError{msg: "string set must only contain non-nil strings"}
}
av.SS = append(av.SS, elem.S)
return nil
}
} else { // List
av.L = make([]*dynamodb.AttributeValue, 0, v.Len())
elemFn = func(elem dynamodb.AttributeValue) error {
av.L = append(av.L, &elem)
return nil
}
}
if n, err := e.encodeList(v, fieldTag, elemFn); err != nil {
return err
} else if n == 0 {
encodeNull(av)
}
}
return nil
}
func (e *Encoder) encodeList(v reflect.Value, fieldTag tag, elemFn func(dynamodb.AttributeValue) error) (int, error) {
count := 0
for i := 0; i < v.Len(); i++ {
elem := dynamodb.AttributeValue{}
err := e.encode(&elem, v.Index(i), tag{OmitEmpty: fieldTag.OmitEmptyElem})
skip, err := keepOrOmitEmpty(fieldTag.OmitEmptyElem, &elem, err)
if err != nil {
return 0, err
} else if skip {
continue
}
if err := elemFn(elem); err != nil {
return 0, err
}
count++
}
return count, nil
}
func (e *Encoder) encodeScalar(av *dynamodb.AttributeValue, v reflect.Value, fieldTag tag) error {
if v.Type() == numberType {
s := v.String()
if fieldTag.AsString {
av.S = &s
} else {
av.N = &s
}
return nil
}
switch v.Kind() {
case reflect.Bool:
av.BOOL = new(bool)
*av.BOOL = v.Bool()
case reflect.String:
if err := e.encodeString(av, v); err != nil {
return err
}
default:
// Fallback to encoding numbers, will return invalid type if not supported
if err := e.encodeNumber(av, v); err != nil {
return err
}
if fieldTag.AsString && av.NULL == nil && av.N != nil {
av.S = av.N
av.N = nil
}
}
return nil
}
func (e *Encoder) encodeNumber(av *dynamodb.AttributeValue, v reflect.Value) error {
if used, err := tryMarshaler(av, v); used {
return err
}
var out string
switch v.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
out = encodeInt(v.Int())
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
out = encodeUint(v.Uint())
case reflect.Float32, reflect.Float64:
out = encodeFloat(v.Float())
default:
return &unsupportedMarshalTypeError{Type: v.Type()}
}
av.N = &out
return nil
}
func (e *Encoder) encodeString(av *dynamodb.AttributeValue, v reflect.Value) error {
if used, err := tryMarshaler(av, v); used {
return err
}
switch v.Kind() {
case reflect.String:
s := v.String()
if len(s) == 0 && e.NullEmptyString {
encodeNull(av)
} else {
av.S = &s
}
default:
return &unsupportedMarshalTypeError{Type: v.Type()}
}
return nil
}
func encodeInt(i int64) string {
return strconv.FormatInt(i, 10)
}
func encodeUint(u uint64) string {
return strconv.FormatUint(u, 10)
}
func encodeFloat(f float64) string {
return strconv.FormatFloat(f, 'f', -1, 64)
}
func encodeNull(av *dynamodb.AttributeValue) {
t := true
*av = dynamodb.AttributeValue{NULL: &t}
}
func valueElem(v reflect.Value) reflect.Value {
switch v.Kind() {
case reflect.Interface, reflect.Ptr:
for v.Kind() == reflect.Interface || v.Kind() == reflect.Ptr {
v = v.Elem()
}
}
return v
}
func emptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
return v.Len() == 0
case reflect.Bool:
return !v.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return v.Int() == 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
case reflect.Interface, reflect.Ptr:
return v.IsNil()
}
return false
}
func tryMarshaler(av *dynamodb.AttributeValue, v reflect.Value) (bool, error) {
if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
v = v.Addr()
}
if v.Type().NumMethod() == 0 {
return false, nil
}
if m, ok := v.Interface().(Marshaler); ok {
return true, m.MarshalDynamoDBAttributeValue(av)
}
return false, nil
}
func keepOrOmitEmpty(omitEmpty bool, av *dynamodb.AttributeValue, err error) (bool, error) {
if err != nil {
if _, ok := err.(*unsupportedMarshalTypeError); ok {
return true, nil
}
return false, err
}
if av.NULL != nil && omitEmpty {
return true, nil
}
return false, nil
}
// An InvalidMarshalError is an error type representing an error
// occurring when marshaling a Go value type to an AttributeValue.
type InvalidMarshalError struct {
emptyOrigError
msg string
}
// Error returns the string representation of the error.
// satisfying the error interface
func (e *InvalidMarshalError) Error() string {
return fmt.Sprintf("%s: %s", e.Code(), e.Message())
}
// Code returns the code of the error, satisfying the awserr.Error
// interface.
func (e *InvalidMarshalError) Code() string {
return "InvalidMarshalError"
}
// Message returns the detailed message of the error, satisfying
// the awserr.Error interface.
func (e *InvalidMarshalError) Message() string {
return e.msg
}
// An unsupportedMarshalTypeError represents a Go value type
// which cannot be marshaled into an AttributeValue and should
// be skipped by the marshaler.
type unsupportedMarshalTypeError struct {
emptyOrigError
Type reflect.Type
}
// Error returns the string representation of the error.
// satisfying the error interface
func (e *unsupportedMarshalTypeError) Error() string {
return fmt.Sprintf("%s: %s", e.Code(), e.Message())
}
// Code returns the code of the error, satisfying the awserr.Error
// interface.
func (e *unsupportedMarshalTypeError) Code() string {
return "unsupportedMarshalTypeError"
}
// Message returns the detailed message of the error, satisfying
// the awserr.Error interface.
func (e *unsupportedMarshalTypeError) Message() string {
return "Go value type " + e.Type.String() + " is not supported"
}

View file

@ -0,0 +1,269 @@
package dynamodbattribute
import (
"reflect"
"sort"
"strings"
)
type field struct {
tag
Name string
NameFromTag bool
Index []int
Type reflect.Type
}
func fieldByName(fields []field, name string) (field, bool) {
foldExists := false
foldField := field{}
for _, f := range fields {
if f.Name == name {
return f, true
}
if !foldExists && strings.EqualFold(f.Name, name) {
foldField = f
foldExists = true
}
}
return foldField, foldExists
}
func buildField(pIdx []int, i int, sf reflect.StructField, fieldTag tag) field {
f := field{
Name: sf.Name,
Type: sf.Type,
tag: fieldTag,
}
if len(fieldTag.Name) != 0 {
f.NameFromTag = true
f.Name = fieldTag.Name
}
f.Index = make([]int, len(pIdx)+1)
copy(f.Index, pIdx)
f.Index[len(pIdx)] = i
return f
}
func unionStructFields(t reflect.Type, opts MarshalOptions) []field {
fields := enumFields(t, opts)
sort.Sort(fieldsByName(fields))
fields = visibleFields(fields)
return fields
}
// enumFields will recursively iterate through a structure and its nested
// anonymous fields.
//
// Based on the enoding/json struct field enumeration of the Go Stdlib
// https://golang.org/src/encoding/json/encode.go typeField func.
func enumFields(t reflect.Type, opts MarshalOptions) []field {
// Fields to explore
current := []field{}
next := []field{{Type: t}}
// count of queued names
count := map[reflect.Type]int{}
nextCount := map[reflect.Type]int{}
visited := map[reflect.Type]struct{}{}
fields := []field{}
for len(next) > 0 {
current, next = next, current[:0]
count, nextCount = nextCount, map[reflect.Type]int{}
for _, f := range current {
if _, ok := visited[f.Type]; ok {
continue
}
visited[f.Type] = struct{}{}
for i := 0; i < f.Type.NumField(); i++ {
sf := f.Type.Field(i)
if sf.PkgPath != "" && !sf.Anonymous {
// Ignore unexported and non-anonymous fields
// unexported but anonymous field may still be used if
// the type has exported nested fields
continue
}
fieldTag := tag{}
fieldTag.parseAVTag(sf.Tag)
if opts.SupportJSONTags && fieldTag == (tag{}) {
fieldTag.parseJSONTag(sf.Tag)
}
if fieldTag.Ignore {
continue
}
ft := sf.Type
if ft.Name() == "" && ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
structField := buildField(f.Index, i, sf, fieldTag)
structField.Type = ft
if !sf.Anonymous || ft.Kind() != reflect.Struct {
fields = append(fields, structField)
if count[f.Type] > 1 {
// If there were multiple instances, add a second,
// so that the annihilation code will see a duplicate.
// It only cares about the distinction between 1 or 2,
// so don't bother generating any more copies.
fields = append(fields, structField)
}
continue
}
// Record new anon struct to explore next round
nextCount[ft]++
if nextCount[ft] == 1 {
next = append(next, structField)
}
}
}
}
return fields
}
// visibleFields will return a slice of fields which are visible based on
// Go's standard visiblity rules with the exception of ties being broken
// by depth and struct tag naming.
//
// Based on the enoding/json field filtering of the Go Stdlib
// https://golang.org/src/encoding/json/encode.go typeField func.
func visibleFields(fields []field) []field {
// Delete all fields that are hidden by the Go rules for embedded fields,
// except that fields with JSON tags are promoted.
// The fields are sorted in primary order of name, secondary order
// of field index length. Loop over names; for each name, delete
// hidden fields by choosing the one dominant field that survives.
out := fields[:0]
for advance, i := 0, 0; i < len(fields); i += advance {
// One iteration per name.
// Find the sequence of fields with the name of this first field.
fi := fields[i]
name := fi.Name
for advance = 1; i+advance < len(fields); advance++ {
fj := fields[i+advance]
if fj.Name != name {
break
}
}
if advance == 1 { // Only one field with this name
out = append(out, fi)
continue
}
dominant, ok := dominantField(fields[i : i+advance])
if ok {
out = append(out, dominant)
}
}
fields = out
sort.Sort(fieldsByIndex(fields))
return fields
}
// dominantField looks through the fields, all of which are known to
// have the same name, to find the single field that dominates the
// others using Go's embedding rules, modified by the presence of
// JSON tags. If there are multiple top-level fields, the boolean
// will be false: This condition is an error in Go and we skip all
// the fields.
//
// Based on the enoding/json field filtering of the Go Stdlib
// https://golang.org/src/encoding/json/encode.go dominantField func.
func dominantField(fields []field) (field, bool) {
// The fields are sorted in increasing index-length order. The winner
// must therefore be one with the shortest index length. Drop all
// longer entries, which is easy: just truncate the slice.
length := len(fields[0].Index)
tagged := -1 // Index of first tagged field.
for i, f := range fields {
if len(f.Index) > length {
fields = fields[:i]
break
}
if f.NameFromTag {
if tagged >= 0 {
// Multiple tagged fields at the same level: conflict.
// Return no field.
return field{}, false
}
tagged = i
}
}
if tagged >= 0 {
return fields[tagged], true
}
// All remaining fields have the same length. If there's more than one,
// we have a conflict (two fields named "X" at the same level) and we
// return no field.
if len(fields) > 1 {
return field{}, false
}
return fields[0], true
}
// fieldsByName sorts field by name, breaking ties with depth,
// then breaking ties with "name came from json tag", then
// breaking ties with index sequence.
//
// Based on the enoding/json field filtering of the Go Stdlib
// https://golang.org/src/encoding/json/encode.go fieldsByName type.
type fieldsByName []field
func (x fieldsByName) Len() int { return len(x) }
func (x fieldsByName) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x fieldsByName) Less(i, j int) bool {
if x[i].Name != x[j].Name {
return x[i].Name < x[j].Name
}
if len(x[i].Index) != len(x[j].Index) {
return len(x[i].Index) < len(x[j].Index)
}
if x[i].NameFromTag != x[j].NameFromTag {
return x[i].NameFromTag
}
return fieldsByIndex(x).Less(i, j)
}
// fieldsByIndex sorts field by index sequence.
//
// Based on the enoding/json field filtering of the Go Stdlib
// https://golang.org/src/encoding/json/encode.go fieldsByIndex type.
type fieldsByIndex []field
func (x fieldsByIndex) Len() int { return len(x) }
func (x fieldsByIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x fieldsByIndex) Less(i, j int) bool {
for k, xik := range x[i].Index {
if k >= len(x[j].Index) {
return false
}
if xik != x[j].Index[k] {
return xik < x[j].Index[k]
}
}
return len(x[i].Index) < len(x[j].Index)
}

View file

@ -0,0 +1,65 @@
package dynamodbattribute
import (
"reflect"
"strings"
)
type tag struct {
Name string
Ignore bool
OmitEmpty bool
OmitEmptyElem bool
AsString bool
AsBinSet, AsNumSet, AsStrSet bool
}
func (t *tag) parseAVTag(structTag reflect.StructTag) {
tagStr := structTag.Get("dynamodbav")
if len(tagStr) == 0 {
return
}
t.parseTagStr(tagStr)
}
func (t *tag) parseJSONTag(structTag reflect.StructTag) {
tagStr := structTag.Get("json")
if len(tagStr) == 0 {
return
}
t.parseTagStr(tagStr)
}
func (t *tag) parseTagStr(tagStr string) {
parts := strings.Split(tagStr, ",")
if len(parts) == 0 {
return
}
if name := parts[0]; name == "-" {
t.Name = ""
t.Ignore = true
} else {
t.Name = name
t.Ignore = false
}
for _, opt := range parts[1:] {
switch opt {
case "omitempty":
t.OmitEmpty = true
case "omitemptyelem":
t.OmitEmptyElem = true
case "string":
t.AsString = true
case "binaryset":
t.AsBinSet = true
case "numberset":
t.AsNumSet = true
case "stringset":
t.AsStrSet = true
}
}
}

View file

@ -0,0 +1,143 @@
// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT.
// Package dynamodbiface provides an interface to enable mocking the Amazon DynamoDB service client
// for testing your code.
//
// It is important to note that this interface will have breaking changes
// when the service model is updated and adds new API operations, paginators,
// and waiters.
package dynamodbiface
import (
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/dynamodb"
)
// DynamoDBAPI provides an interface to enable mocking the
// dynamodb.DynamoDB service client's API operation,
// paginators, and waiters. This make unit testing your code that calls out
// to the SDK's service client's calls easier.
//
// The best way to use this interface is so the SDK's service client's calls
// can be stubbed out for unit testing your code with the SDK without needing
// to inject custom request handlers into the the SDK's request pipeline.
//
// // myFunc uses an SDK service client to make a request to
// // Amazon DynamoDB.
// func myFunc(svc dynamodbiface.DynamoDBAPI) bool {
// // Make svc.BatchGetItem request
// }
//
// func main() {
// sess := session.New()
// svc := dynamodb.New(sess)
//
// myFunc(svc)
// }
//
// In your _test.go file:
//
// // Define a mock struct to be used in your unit tests of myFunc.
// type mockDynamoDBClient struct {
// dynamodbiface.DynamoDBAPI
// }
// func (m *mockDynamoDBClient) BatchGetItem(input *dynamodb.BatchGetItemInput) (*dynamodb.BatchGetItemOutput, error) {
// // mock response/functionality
// }
//
// TestMyFunc(t *testing.T) {
// // Setup Test
// mockSvc := &mockDynamoDBClient{}
//
// myfunc(mockSvc)
//
// // Verify myFunc's functionality
// }
//
// It is important to note that this interface will have breaking changes
// when the service model is updated and adds new API operations, paginators,
// and waiters. Its suggested to use the pattern above for testing, or using
// tooling to generate mocks to satisfy the interfaces.
type DynamoDBAPI interface {
BatchGetItemRequest(*dynamodb.BatchGetItemInput) (*request.Request, *dynamodb.BatchGetItemOutput)
BatchGetItem(*dynamodb.BatchGetItemInput) (*dynamodb.BatchGetItemOutput, error)
BatchGetItemPages(*dynamodb.BatchGetItemInput, func(*dynamodb.BatchGetItemOutput, bool) bool) error
BatchWriteItemRequest(*dynamodb.BatchWriteItemInput) (*request.Request, *dynamodb.BatchWriteItemOutput)
BatchWriteItem(*dynamodb.BatchWriteItemInput) (*dynamodb.BatchWriteItemOutput, error)
CreateTableRequest(*dynamodb.CreateTableInput) (*request.Request, *dynamodb.CreateTableOutput)
CreateTable(*dynamodb.CreateTableInput) (*dynamodb.CreateTableOutput, error)
DeleteItemRequest(*dynamodb.DeleteItemInput) (*request.Request, *dynamodb.DeleteItemOutput)
DeleteItem(*dynamodb.DeleteItemInput) (*dynamodb.DeleteItemOutput, error)
DeleteTableRequest(*dynamodb.DeleteTableInput) (*request.Request, *dynamodb.DeleteTableOutput)
DeleteTable(*dynamodb.DeleteTableInput) (*dynamodb.DeleteTableOutput, error)
DescribeLimitsRequest(*dynamodb.DescribeLimitsInput) (*request.Request, *dynamodb.DescribeLimitsOutput)
DescribeLimits(*dynamodb.DescribeLimitsInput) (*dynamodb.DescribeLimitsOutput, error)
DescribeTableRequest(*dynamodb.DescribeTableInput) (*request.Request, *dynamodb.DescribeTableOutput)
DescribeTable(*dynamodb.DescribeTableInput) (*dynamodb.DescribeTableOutput, error)
GetItemRequest(*dynamodb.GetItemInput) (*request.Request, *dynamodb.GetItemOutput)
GetItem(*dynamodb.GetItemInput) (*dynamodb.GetItemOutput, error)
ListTablesRequest(*dynamodb.ListTablesInput) (*request.Request, *dynamodb.ListTablesOutput)
ListTables(*dynamodb.ListTablesInput) (*dynamodb.ListTablesOutput, error)
ListTablesPages(*dynamodb.ListTablesInput, func(*dynamodb.ListTablesOutput, bool) bool) error
ListTagsOfResourceRequest(*dynamodb.ListTagsOfResourceInput) (*request.Request, *dynamodb.ListTagsOfResourceOutput)
ListTagsOfResource(*dynamodb.ListTagsOfResourceInput) (*dynamodb.ListTagsOfResourceOutput, error)
PutItemRequest(*dynamodb.PutItemInput) (*request.Request, *dynamodb.PutItemOutput)
PutItem(*dynamodb.PutItemInput) (*dynamodb.PutItemOutput, error)
QueryRequest(*dynamodb.QueryInput) (*request.Request, *dynamodb.QueryOutput)
Query(*dynamodb.QueryInput) (*dynamodb.QueryOutput, error)
QueryPages(*dynamodb.QueryInput, func(*dynamodb.QueryOutput, bool) bool) error
ScanRequest(*dynamodb.ScanInput) (*request.Request, *dynamodb.ScanOutput)
Scan(*dynamodb.ScanInput) (*dynamodb.ScanOutput, error)
ScanPages(*dynamodb.ScanInput, func(*dynamodb.ScanOutput, bool) bool) error
TagResourceRequest(*dynamodb.TagResourceInput) (*request.Request, *dynamodb.TagResourceOutput)
TagResource(*dynamodb.TagResourceInput) (*dynamodb.TagResourceOutput, error)
UntagResourceRequest(*dynamodb.UntagResourceInput) (*request.Request, *dynamodb.UntagResourceOutput)
UntagResource(*dynamodb.UntagResourceInput) (*dynamodb.UntagResourceOutput, error)
UpdateItemRequest(*dynamodb.UpdateItemInput) (*request.Request, *dynamodb.UpdateItemOutput)
UpdateItem(*dynamodb.UpdateItemInput) (*dynamodb.UpdateItemOutput, error)
UpdateTableRequest(*dynamodb.UpdateTableInput) (*request.Request, *dynamodb.UpdateTableOutput)
UpdateTable(*dynamodb.UpdateTableInput) (*dynamodb.UpdateTableOutput, error)
WaitUntilTableExists(*dynamodb.DescribeTableInput) error
WaitUntilTableNotExists(*dynamodb.DescribeTableInput) error
}
var _ DynamoDBAPI = (*dynamodb.DynamoDB)(nil)

View file

@ -0,0 +1,110 @@
// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT.
package dynamodb
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/client/metadata"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/aws/aws-sdk-go/private/protocol/jsonrpc"
)
// Amazon DynamoDB is a fully managed NoSQL database service that provides fast
// and predictable performance with seamless scalability. DynamoDB lets you
// offload the administrative burdens of operating and scaling a distributed
// database, so that you don't have to worry about hardware provisioning, setup
// and configuration, replication, software patching, or cluster scaling.
//
// With DynamoDB, you can create database tables that can store and retrieve
// any amount of data, and serve any level of request traffic. You can scale
// up or scale down your tables' throughput capacity without downtime or performance
// degradation, and use the AWS Management Console to monitor resource utilization
// and performance metrics.
//
// DynamoDB automatically spreads the data and traffic for your tables over
// a sufficient number of servers to handle your throughput and storage requirements,
// while maintaining consistent and fast performance. All of your data is stored
// on solid state disks (SSDs) and automatically replicated across multiple
// Availability Zones in an AWS region, providing built-in high availability
// and data durability.
// The service client's operations are safe to be used concurrently.
// It is not safe to mutate any of the client's properties though.
// Please also see https://docs.aws.amazon.com/goto/WebAPI/dynamodb-2012-08-10
type DynamoDB struct {
*client.Client
}
// Used for custom client initialization logic
var initClient func(*client.Client)
// Used for custom request initialization logic
var initRequest func(*request.Request)
// Service information constants
const (
ServiceName = "dynamodb" // Service endpoint prefix API calls made to.
EndpointsID = ServiceName // Service ID for Regions and Endpoints metadata.
)
// New creates a new instance of the DynamoDB client with a session.
// If additional configuration is needed for the client instance use the optional
// aws.Config parameter to add your extra config.
//
// Example:
// // Create a DynamoDB client from just a session.
// svc := dynamodb.New(mySession)
//
// // Create a DynamoDB client with additional configuration
// svc := dynamodb.New(mySession, aws.NewConfig().WithRegion("us-west-2"))
func New(p client.ConfigProvider, cfgs ...*aws.Config) *DynamoDB {
c := p.ClientConfig(EndpointsID, cfgs...)
return newClient(*c.Config, c.Handlers, c.Endpoint, c.SigningRegion, c.SigningName)
}
// newClient creates, initializes and returns a new service client instance.
func newClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegion, signingName string) *DynamoDB {
svc := &DynamoDB{
Client: client.New(
cfg,
metadata.ClientInfo{
ServiceName: ServiceName,
SigningName: signingName,
SigningRegion: signingRegion,
Endpoint: endpoint,
APIVersion: "2012-08-10",
JSONVersion: "1.0",
TargetPrefix: "DynamoDB_20120810",
},
handlers,
),
}
// Handlers
svc.Handlers.Sign.PushBackNamed(v4.SignRequestHandler)
svc.Handlers.Build.PushBackNamed(jsonrpc.BuildHandler)
svc.Handlers.Unmarshal.PushBackNamed(jsonrpc.UnmarshalHandler)
svc.Handlers.UnmarshalMeta.PushBackNamed(jsonrpc.UnmarshalMetaHandler)
svc.Handlers.UnmarshalError.PushBackNamed(jsonrpc.UnmarshalErrorHandler)
// Run custom client initialization if present
if initClient != nil {
initClient(svc.Client)
}
return svc
}
// newRequest creates a new request for a DynamoDB operation and runs any
// custom request initialization.
func (c *DynamoDB) newRequest(op *request.Operation, params, data interface{}) *request.Request {
req := c.NewRequest(op, params, data)
// Run custom request initialization if present
if initRequest != nil {
initRequest(req)
}
return req
}

View file

@ -0,0 +1,67 @@
// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT.
package dynamodb
import (
"github.com/aws/aws-sdk-go/private/waiter"
)
// WaitUntilTableExists uses the DynamoDB API operation
// DescribeTable to wait for a condition to be met before returning.
// If the condition is not meet within the max attempt window an error will
// be returned.
func (c *DynamoDB) WaitUntilTableExists(input *DescribeTableInput) error {
waiterCfg := waiter.Config{
Operation: "DescribeTable",
Delay: 20,
MaxAttempts: 25,
Acceptors: []waiter.WaitAcceptor{
{
State: "success",
Matcher: "path",
Argument: "Table.TableStatus",
Expected: "ACTIVE",
},
{
State: "retry",
Matcher: "error",
Argument: "",
Expected: "ResourceNotFoundException",
},
},
}
w := waiter.Waiter{
Client: c,
Input: input,
Config: waiterCfg,
}
return w.Wait()
}
// WaitUntilTableNotExists uses the DynamoDB API operation
// DescribeTable to wait for a condition to be met before returning.
// If the condition is not meet within the max attempt window an error will
// be returned.
func (c *DynamoDB) WaitUntilTableNotExists(input *DescribeTableInput) error {
waiterCfg := waiter.Config{
Operation: "DescribeTable",
Delay: 20,
MaxAttempts: 25,
Acceptors: []waiter.WaitAcceptor{
{
State: "success",
Matcher: "error",
Argument: "",
Expected: "ResourceNotFoundException",
},
},
}
w := waiter.Waiter{
Client: c,
Input: input,
Config: waiterCfg,
}
return w.Wait()
}