AWS Lambda（Python） の開発環境・テスト・デプロイ・CI 考察

.
├── buildspec.yml
├── deploy.sh
├── docker-compose.yml
├── environments
│   ├── common.sh
│   └── sam-local.json
├── integration_test.sh
├── requirements.txt
├── src
│   └── functions
│        └── heroes
│           ├── index.py
│           └── utils.py
├── test
│    └── functions
└── template_heroes.yaml

python3 -m venv .
source bin/activate.fish # fish shell の場合

pip install -r requirements.txt

pip install boto3
pip freeze > requirements.txt

from datetime import datetime
import decimal
import json

def epoc_by_second_precision(time: datetime):
    return decimal.Decimal(time.replace(microsecond=0).timestamp())

from src.functions.heroes.utils import *

def test_epoc_by_second_precision():
    tstr = '2012-12-29T13:49:37+0900'
    tdatetime = datetime.strptime(tstr, '%Y-%m-%dT%H:%M:%S%z')
    sut = epoc_by_second_precision(tdatetime)

    assert int(sut) == 1356756577

python -m pytest

platform darwin -- Python 3.6.2, pytest-3.4.2, py-1.5.2, pluggy-0.6.0
rootdir: /Users/wada.yusuke/.ghq/github.com/cm-wada-yusuke/python-lambda-template, inifile:
collected 1 item

test/functions/heroes/test_utils.py .

version: "3.3"

services:
  localstack:
    container_name: localstack
    image: localstack/localstack
    ports:
      - "8080:8080"
      - "4569:4569"
    environment:
      - SERVICES=dynamodb
      - DEFAULT_REGION=ap-northeast-1
      - DOCKER_HOST=unix:///var/run/docker.sock

docker-compose up -d

import boto3
import datetime
import uuid
from builtins import Exception
import os
from src.functions.heroes.utils import *

DYNAMODB_ENDPOINT = os.getenv('DYNAMODB_ENDPOINT')
HERO_TABLE_NAME = os.getenv('HERO_TABLE_NAME')
DYNAMO = boto3.resource(
    'dynamodb',
    endpoint_url=DYNAMODB_ENDPOINT
)
DYNAMODB_TABLE = DYNAMO.Table(HERO_TABLE_NAME)

def get(event, context):
    try:
        hero_id = event['id']
        dynamo_response = DYNAMODB_TABLE.get_item(
            Key={
                'id': hero_id
            }
        )
        response = json.dumps(dynamo_response['Item'], cls=DecimalEncoder, ensure_ascii=False)
        return response
    except Exception as error:
        raise error

{
  "id": "test-id"
}

AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: Simple CRUD webservice. State is stored in a SimpleTable (DynamoDB) resource.
Parameters:
  Env:
    Type: String
    Default: local
  DynamoDBEndpoint:
    Type: String
    Default: https://dynamodb.ap-northeast-1.amazonaws.com/
  HeroTableName:
    Type: String
    Default: CM-Heroes
  BucketName:
    Type: String
    Default: hero-lambda-deploy
  CodeKey:
    Type: String
    Default: heroes/0000.zip
Resources:
  GetHeroes:
    Type: AWS::Serverless::Function
    Properties:
      FunctionName:
        Fn::Sub: ${Env}-heroes-GetHeroes
      Handler: src/functions/heroes/index.get
      Runtime: python3.6
      CodeUri:
          Bucket: !Ref BucketName
          Key: !Ref CodeKey
      Policies: AmazonDynamoDBReadOnlyAccess
      Environment:
        Variables:
          ENV: !Ref Env
          DYNAMODB_ENDPOINT: !Ref DynamoDBEndpoint
          HERO_TABLE_NAME:
            Fn::Sub: ${Env}-${HeroTableName}

{
  "GetHeroes": {
    "DYNAMODB_ENDPOINT": "http://localstack:4569/",
    "HERO_TABLE_NAME": "CM-Heroes"
  }
}

sam local invoke \
--docker-network a27c0476cb8e \
-t template_heroes.yaml \
--event test/functions/heroes/examples/get_payload.json \
--env-vars environments/sam-local.json GetHeroes


2018/03/09 11:32:42 Successfully parsed template_heroes.yaml
...省略...

{"name": "Test-man", "created_at": 1520400553, "id": "test-id", "office": "virtual", "updated_at": 1520400554}

#!/usr/bin/env bats

. environments/common.sh

setup() {
    echo "setup"
    docker_network_id=`docker network ls -q -f NAME=$DOCKER_NAME`
}

teardown() {
    echo "teardown"
}

@test "DynamoDB Get Hero Function response the correct item" {
    # テストデータを用意
    data='{
        "id": {"S": "test-id"},
        "name": {"S": "Test-man"},
        "created_at": {"N": "1520400553"},
        "updated_at": {"N": "1520400554"},
        "office": {"S": "virtual"} }'

    expected=`echo "${data}" | jq -r .`

    # テストデータを LocalStack の DynamoDB に投入
    aws --endpoint-url=http://localhost:4569 dynamodb put-item --table-name CM-Heroes --item "${data}"

    # SAM Local を起動し、Lambda Function の出力を得る
    actual=`sam local invoke --docker-network ${docker_network_id} -t template_heroes.yaml --event test/functions/heroes/examples/get_payload.json --env-vars environments/sam-local.json GetHeroes | jq -r .`

    # 出力内容をテスト
    [ `echo "${actual}" | jq .id` = `echo "${expected}" | jq .id.S` ]
    [ `echo "${actual}" | jq .name` = `echo "${expected}" | jq .name.S` ]
    [ `echo "${actual}" | jq .created_at` -eq  `echo "${expected}" | jq .created_at.N | bc` ]
    [ `echo "${actual}" | jq .updated_at` -eq `echo "${expected}" | jq .updated_at.N | bc` ]
    [ `echo "${actual}" | jq .office` = `echo "${expected}" | jq .office.S` ]
}

bats test/functions/heroes/integration.bats

✓ DynamoDB Get Hero Function response the correct item

1 tests, 0 failures

#!/usr/bin/env bash

. environments/common.sh

if [ -z $ENV ]; then
  echo "Set the environment name. Such as test, stg, and prd." 1>&2
  exit 1
fi

# 必要ファイルを集め、ZIPを作成する
pip install -r requirements.txt -t deploy
cp -R src deploy
cd deploy
zip -r source.zip *
hash=`openssl md5 source.zip | awk '{print $2}'`
echo "source.zip: hash = $hash"
filename="${hash}.zip"
mv source.zip $filename

bucket=$DEPLOY_S3_BUCKET

for item in ${FUNCTION_GROUP[@]} ; do

    # S3 へアップロード
    s3_keyname="${item}/${filename}"
    aws s3 cp $filename  s3://${bucket}/${item}/

    # CloudFormation テンプレート作成
    cp ../template_${item}.yaml ./
    aws cloudformation package \
        --template-file template_${item}.yaml \
        --s3-bucket ${bucket} \
        --output-template-file packaged-${item}.yaml

    # CloudFormation によるデプロイ
    aws cloudformation deploy \
        --template-file packaged-${item}.yaml \
        --stack-name ${ENV}-${item}-lambda  \
        --capabilities CAPABILITY_IAM \
        --parameter-overrides \
            Env=${ENV} \
            CodeKey=${s3_keyname}
done
cd ..
rm -r deploy/

  
version: 0.2

env:
  variables:
    DOCKER_COMPOSE_VERSION: "1.18.0"

phases:
  install:
    commands:
      - sudo apt-get update
      - sudo apt-get install zip bc
      - sudo curl -o /usr/local/bin/jq -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && sudo chmod +x /usr/local/bin/jq
      - curl -sL https://deb.nodesource.com/setup_8.x | sudo -E bash -
      - sudo apt-get install -y nodejs
      - sudo npm cache clean --force
      - sudo npm install -g n
      - sudo n stable
      - sudo ln -sf /usr/local/bin/node /usr/bin/node
      - sudo apt-get purge -y nodejs
      - npm update -g npm
      - npm -g install --unsafe-perm aws-sam-local
      - python -m venv .
      - . bin/activate
      - pip install --upgrade pip
      - pip install --upgrade awscli
      - curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose
      - git clone https://github.com/sstephenson/bats.git
      - cd bats
      - sudo ./install.sh /usr/local
      - cd ../
  pre_build:
    commands:
      - pip install -r requirements.txt
      - docker-compose up -d
  build:
    commands:
      - python -m pytest      # Unit Test
      - ./integration_test.sh # 結合テスト
  post_build:
    commands:
      - ./deploy.sh           # ビルド & アップロード & デプロイ

...前略...
def put(event, context):
    try:
        hero_id = str(uuid.uuid4())

        name = event.get('name')
        office = event.get('office')
        updated_at = epoc_by_second_precision(datetime.now())

        response = DYNAMODB_TABLE.put_item(
            Item={
                'id': hero_id,
                'name': name,
                'office': office,
                'updated_at': updated_at,
                'created_at': updated_at,
            }
        )
        return response
    except Exception as error:
        raise error

...前略...
PutHeroes:
  Type: AWS::Serverless::Function
  Properties:
    FunctionName:
      Fn::Sub: ${Env}-heroes-PutHeroes
    Handler: src/functions/heroes/index.put
    Runtime: python3.6
    CodeUri:
        Bucket: !Ref BucketName
        Key: !Ref CodeKey
    Policies: AmazonDynamoDBFullAccess
    Environment:
      Variables:
        ENV: !Ref Env
        DYNAMODB_ENDPOINT: !Ref DynamoDBEndpoint
        HERO_TABLE_NAME:
          Fn::Sub: ${Env}-${HeroTableName}

{
  "GetHeroes": {
    "DYNAMODB_ENDPOINT": "http://localstack:4569/",
    "HERO_TABLE_NAME": "CM-Heroes"
  },
  "PutHeroes": {
    "DYNAMODB_ENDPOINT": "http://localstack:4569/",
    "HERO_TABLE_NAME": "CM-Heroes"
  }
}

...前略...
@test "DynamoDB PUT Hero Function response code is 200" {
    result=`sam local invoke --docker-network ${docker_network_id} -t template_heroes.yaml --event test/functions/heroes/examples/put_payload.json --env-vars environments/sam-local.json PutHeroes | jq '.ResponseMetadata.HTTPStatusCode'`
    [ $result -eq 200 ]
}

├── src
   └── functions
        ├── heroes
        │   ├── index.py
        │   └── utils.py
        └── offices
            └── index.py

find test -name '*.bats' | xargs bats

1..4
ok 1 DynamoDB Get Hero Function response the correct item
ok 2 DynamoDB PUT Hero Function response code is 200
ok 3 DynamoDB Get Office Function response the correct item
ok 4 DynamoDB PUT Office Function response code is 200