【AWS/Serverless Framework】 S3rver & Serverless Offineで構築するローカルのS3 bucketライクなストレージ環境の導入する方法

            
            FROM node:10.18-alpine
ENV NODE_ENV development
WORKDIR /usr/src/app

#awscli on dokcer alpine
ARG pip_installer="https://bootstrap.pypa.io/get-pip.py"
ARG awscli_version="1.16.307"

#Install dependent packages
RUN apk update && apk upgrade && apk add --no-cache \
    bash git openssh \
    python \
    curl \
    groff \
    jq \
    less

#Install awscli
RUN curl ${pip_installer} | python && pip install awscli==${awscli_version}

#Completing input
RUN bash -c 'echo complete -C '/usr/bin/aws_completer' aws >> $HOME/.bashrc'
ENV PS1="[\u@\h:\w]$"
        

            
            version: '3'

services:
    app: # appというサービス名で構築
        image: my/s3rver_dckr # コンテナー名
        build: .
        environment:
            NODE_ENV: development
            AWS_ACCESS_KEY_ID: "ABCDEFGHIJKLMNOPQRST" # IAMのユーザーID (ローカルだとなんでも動きます)
            AWS_SECRET_ACCESS_KEY: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # IAMのユーザーパスワード (ローカルだとなんでも動きます)
            AWS_DEFAULT_REGION: "us-east-1"
        ports:
            - "6122:6122" # S3rverのサービス用に6122番を指定
        volumes:
            - ./:/usr/src/app
        tty: true
        

            
            % docker-compose up -d && docker-compose exec app bash
        

            
            % yarn add s3rver -D
        

            
            {
    "name": "s3rver_dckr",
    "version": "0.0.1",
    "scripts": {
        "s3:start": "s3rver --address 0.0.0.0 --port 6122 --directory /usr/src/app/tmp"
    },
    // ...
    "devDependencies": {
        "s3rver": "^3.5.0"
    }
}
        

            
            $ cat test/test.txt
1234
$ cat test/test2.txt
5678
        

            
            $ tree -I node_modules
.
├── Dockerfile
├── README.md
├── docker-compose.yml
├── package.json
├── test
│   ├── test.txt
│   └── test2.txt
├── tmp
└── yarn.lock
        

            
            $ yarn s3:start
        

            
            #バケットの作成
#例) hogeという名前の空バケットを作成
$ curl -XPUT 'http://localhost:6122/hoge'

#バケットの中身リスト表示
#例) hogeという名前のバケットの中身を表示
#(応答はXML形式のため、xmllintなどでフォーマット表示するのがよろしいかと思います)
$ curl -XGET 'http://localhost:6122/hoge/' | xmllint --format -

#バケットの削除 (※空バケットに限る)
#例) hogeという名前の空バケットを削除
$ curl -XDELETE 'http://localhost:6122/hoge'

#ファイルのアップロード
#例) test.txtをhogeにアップロード
$ curl -XPUT -T test/test.txt 'http://localhost:6122/hoge/'
#ただし、同時に二つ以上のファイルはアップロードできない
$ curl -XPUT -T test/test.txt -T test/test2.txt 'http://localhost:6122/hoge/'

#ファイルの取得
#例) test2.txtをhogeから取得(応答はファイルの中身)
$ curl -XGET 'http://localhost:6122/hoge/test2.txt'
5678

#ファイルの削除
#例) test.txtをhogeから削除
$ curl -XDELETE 'http://localhost:6122/hoge/test.txt'
        

            
            $ touch inject-command.yml
$ tree -I node_modules
.
├── Dockerfile
├── README.md
├── docker-compose.yml
├── inject-command.yml
├── package.json
├── test
├── tmp
└── yarn.lock
        

            
            version: '3'

services:
    app:
        entrypoint: yarn s3:start
        restart: always
        

            
            $ docker-compose -f docker-compose.yml \
    -f inject-command.yml \
    up -d
        

            
            $ tree -I 'node_modules'
.
├── Dockerfile
├── docker-compose.yml
├── package.json
├── app
│   ├── handler.js
│   ├── package.json
│   ├── serverless.yml
│   └── webpack.config.js
└── yarn.lock
        

            
            FROM node:10.18-alpine
ENV NODE_ENV development
WORKDIR /usr/src/app

#awscli on dokcer alpine
ARG pip_installer="https://bootstrap.pypa.io/get-pip.py"
ARG awscli_version="1.18.31" #👈最新バージョンを手動で書き換えください

#Install dependent packages
RUN apk update && apk upgrade && apk add --no-cache \
    bash git openssh \
    python \
    curl \
    groff \
    jq \
    less

#Install awscli
RUN curl ${pip_installer} | python && pip install awscli==${awscli_version}

#Completing input
RUN bash -c 'echo complete -C '/usr/bin/aws_completer' aws >> $HOME/.bashrc'
ENV PS1="[\u@\h:\w]$"
        

            
            version: '3'

services:
    app:
        image: tm/sls_dckr_s3_local
        build: .
        environment:
            NODE_ENV: development
            AWS_ACCESS_KEY_ID: "ABCDEFGHIJKLMNOPQRST" # IAMのユーザーID (本番環境Deploy用)
            AWS_SECRET_ACCESS_KEY: "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" # IAMのユーザーパスワード (本番環境Deploy用)
            AWS_DEFAULT_REGION: "us-east-1"
            LOCAL_SLS_SERVICE_NAME: "my-sls-local-s3"
        ports:
            - "6121:6121"
        volumes:
            - ./:/usr/src/app
        tty: true
        

            
            #S3rver のコンテナIDをチェック
% docker ps -a
CONTAINER ID        IMAGE                          COMMAND                  CREATED             STATUS                      PORTS                                                      NAMES
736ba12e4d4b        tm/s3rver_dckr                 "yarn s3:start"          24 hours ago        Up 5 hours                  0.0.0.0:6122->6122/tcp                                     s3rver-local-dckr_app_1

#コンテナIDからIPを取得
% docker inspect --format='{{range .NetworkSettings.Networks}}{{.Gateway}}{{end}}' 736
172.26.0.1
        

            
            % touch env-s3-local.yml
        

            
            version: '3'

services:
  app:
    environment:
      OUTER_S3_ID: "172.26.0.1" # PLEASE MAKE SURE OF IT EVERYTIME
        

            
            % docker-compose -f docker-compose.yml \
    -f env-s3-local.yml up -d \
    && docker-compose exec app bash
        

            
            % tree -I 'node_modules'
.
├── Dockerfile
├── README.md
├── app
│   ├── config
│   │   ├── bucket_method.yml
│   │   └── obj_method.yml
│   ├── handlers
│   │   ├── bucket_ctrl.js
│   │   └── obj_ctrl.js
│   ├── package.json
│   ├── serverless.yml
│   ├── webpack.config.js
│   └── yarn.lock
├── docker-compose.yml
├── env-s3-local.yml
├── package.json
└── yarn.lock
        

            
            "use strict";

const AWS = require('aws-sdk');

module.exports.showBucket = (event, context, callback) => {
    const bn = event.pathParameters.bucketname; // Name of bucket

    const s3 = new AWS.S3({
        apiVersion: '2006-03-01',
        s3ForcePathStyle: true,
        accessKeyId: 'S3RVER', // This specific id is required when working offline
        secretAccessKey: 'S3RVER', // This specific key is required when working offline
        endpoint: new AWS.Endpoint(`http://${process.env.OUTER_S3_ID}:6122`)
    });

    const params = {
        Bucket: bn
    };
    s3.listObjects(params, (err, data) => {
        let response = {statusCode: null, body: null};
        if (err) {
            console.log(err);
            response.statusCode = 500;
            response.body = JSON.stringify({code: 500, message: "ListObject Error"});
        } else {
            response.statusCode = 200;
            response.body = JSON.stringify({data: data});
        }
        callback(null, response);
    });
};
        

            
            "use strict";

const AWS = require('aws-sdk');

module.exports.fetchObj = (event, context, callback) => {
    const bn = event.pathParameters.bucketname; // Name of bucket
    const on = event.pathParameters.objname; // Name of object
    const s3 = new AWS.S3({
        apiVersion: '2006-03-01',
        s3ForcePathStyle: true,
        accessKeyId: 'S3RVER', // This specific key is required when working offline
        secretAccessKey: 'S3RVER', // This specific key is required when working offline
        endpoint: new AWS.Endpoint(`http://${process.env.OUTER_S3_ID}:6122`)
    });
    const params = {
        Bucket: bn,
        Key: on,
    };
    s3.getObject(params, (err, data) => {
        let response = {statusCode: null, body: null};
        if (err) {
            console.log(err);
            response.statusCode = 500;
            response.body = JSON.stringify({code: 500, message: "GetObject Error"});
        } else {
            response.statusCode = 200;
            response.body = JSON.stringify({data: data});
        }
        callback(null, response);
    });
};
        

            
            show_bucket:
    - http:
        path: /{bucketname}
        method: get
        

            
            fetch_obj:
    - http:
        path: /{bucketname}/{objname}
        method: get
        

            
            service:
    name: my-sls-local-s3

plugins:
    - serverless-webpack
    - serverless-offline

provider:
    name: aws
    runtime: nodejs10.x
    region: us-east-1
    stage: dev

custom:
    serverless-offline:
        host: 0.0.0.0
        port: 6121

functions:
    showBucket:
        handler: handlers/bucket_ctrl.showBucket
        events: ${file(./config/bucket_method.yml):show_bucket}
    fetchObj:
        handler: handlers/obj_ctrl.fetchObj
        events: ${file(./config/obj_method.yml):fetch_obj}
        

            
            {
    "name": "dckr_sls_s3_local",
    "version": "0.0.1",
    "scripts": {
        "start": "cd ./app && sls offline" // 👈 追加
    },
    "devDependencies": {
        "serverless": "^1.59.0"
    }
}
        

            
            % yarn start
yarn run v1.21.1
$ cd ./app && sls offline
Serverless: Load command interactiveCli

...(中略)

[offline] fetchObj runtime nodejs10.x
Serverless: Routes for fetchObj:
Serverless: GET /{bucketname}/{objname}
[offline] Response Content-Type application/json
Serverless: POST /{apiVersion}/functions/my-sls-local-s3-dev-fetchObj/invocations

Serverless: Offline [HTTP] listening on http://0.0.0.0:6121
Serverless: Enter "rp" to replay the last request
        

            
            #バケット'hoge'の中身を表示
% curl -XGET 'http://localhost:6121/hoge' | jq .

  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   311  100   311    0     0    711      0 --:--:-- --:--:-- --:--:--   710
{
  "data": {
    "IsTruncated": false,
    "Marker": "",
    "Contents": [
      {
        "Key": "test.txt", # 👈 test.txtというファイルがあるのを確認
        "LastModified": "2019-12-20T16:34:26.000Z",
        "ETag": "\"e7df7cd2ca07f4f1ab415d457a6e1c13\"",
        "Size": 5,
        "StorageClass": "STANDARD",
        "Owner": {
          "DisplayName": "S3rver",
          "ID": "123456789000"
        }
      }
    ],
    "Name": "hoge",
    "Prefix": "",
    "MaxKeys": 1000,
    "CommonPrefixes": []
  }
}
        

            
            #'hoge'バケットからファイルを取得
% curl -XGET 'http://localhost:6121/hoge/test.txt' | jq .
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   238  100   238    0     0    746      0 --:--:-- --:--:-- --:--:--   746
{
  "data": {
    "AcceptRanges": "bytes",
    "LastModified": "2019-12-20T16:34:26.000Z",
    "ContentLength": 5,
    "ETag": "\"e7df7cd2ca07f4f1ab415d457a6e1c13\"",
    "ContentType": "binary/octet-stream",
    "Metadata": {},
    "Body": {
      "type": "Buffer",
      "data": [
        49,
        50,
        51,
        52,
        10
      ]
    }
  }
}
        

            
            % touch inject-command.yml
        

            
            version: '3'

services:
    app:
        entrypoint: yarn start
        restart: always # 👈コンテナサービスの永続化
        

            
            % docker-compose -f docker-compose.yml \
    -f env-s3-local.yml \
    -f inject-command.yml \
    up -d