【puppeteerでスクレイピング】Alpine Dockerコンテナからpuppeteerで動的サイトのスクレイピングを試してみる

            FROM alpine:edge

RUN apk update && apk upgrade && \
    apk add --no-cache bash openssh expect

#Dependancies of puppeteer in alpine container
RUN apk add --no-cache \
      chromium \
      nss \
      freetype \
      freetype-dev \
      harfbuzz \
      ca-certificates \
      ttf-freefont \
      nodejs \
      nodejs-npm \
      yarn

RUN npm i -g typescript @babel/core @babel/node ts-node

RUN addgroup -g 1000 -S pptruser && \
    adduser -D -u 1000 -S -G pptruser pptruser
RUN echo '%pptruser ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers

CMD ["bash"]

        

            version: '3'

services:
  app:
    image: puppeteer-alpine:edge
    build: .
    user: "pptruser:pptruser"
    container_name: puppeteer-alpine
    environment:
      NODE_ENV: "development"
      PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: "true"
      PUPPETEER_EXECUTABLE_PATH: "/usr/bin/chromium-browser"
    volumes:
      - ./:/usr/src/app
    working_dir: "/usr/src/app"

        

            $ docker-compose build
$ docker-compose run --rm app bash
[node@d4f30154b2f7:/usr/src/app]$

        

            {
    "name": "puppeteer_ajax",
    "version": "0.1.0",
    "description": "To learn puppetter with docker alpine",
    "main": "./dist/index.js",
    "types": "./dist/index.d.ts",
    "scripts": {
        "build": "tsc",
        "tap": "babel-node dist/index.js",
        "start": "yarn build && yarn tap"
    },
    "devDependencies": {
        "@types/node": "^13.7.1",
        "@types/puppeteer": "^5.4.0"
    },
    "dependencies": {
        "puppeteer": "^5.4.1"
    }
}

        

            $ tree -I node_modules
.
├── Dockerfile
├── docker-compose.yml
├── index.ts
├── package.json
├── src
│   └── index.ts
├── tsconfig.json
└── yarn.lock

        

            {
  "compilerOptions": {
    "target": "es6",
    "module": "commonjs",
    "declaration": true,
    "sourceMap": true,
    "outDir": "./dist",
    "strict": true,
    "moduleResolution": "node",
    "esModuleInterop": true
  },
  "include": [
    "./src/index.ts",
  ],
  "exclude": [
    "./test/*.spec.ts"
  ],
  "compileOnSave": false
}

        

            import puppeteer from 'puppeteer';

(async () => {
    try {
        const browser = await puppeteer.launch({
            executablePath: '/usr/bin/chromium-browser',
            args: ['--disable-dev-shm-usage', '--no-sandbox']
        });
        const page = await browser.newPage();
        await page.goto('https://www.google.com/');
        await page.screenshot({
            path: '/usr/src/app/result.png'
        });
        await browser.close();
    } catch (e) {
        console.error(e);
    }
})();

        

            $ yarn start

        

            FROM alpine:edge
#...中略
RUN apk add --no-cache font-noto-cjk unifont

        

            import puppeteer from 'puppeteer';

(async () => {
    try {
        const browser = await puppeteer.launch({
            executablePath: '/usr/bin/chromium-browser',
            args: ['--disable-dev-shm-usage', '--no-sandbox']
        });
        const page = await browser.newPage();
        await page.goto('https://trade.m******.co.jp/mgap/login');

        //👇①input要素のname属性を利用したフォームの入力
        await page.type('input[name="user"]', 'xxxxxxxx');
        await page.type('input[name="password"]', 'xxxxxxxxxxxxx');

        //👇②idがsubmit-btnのボタン要素をクリック
        page.click('#submit-btn');

        //👇③クリック後にページが遷移するまで待機
        await page.waitForNavigation({
            waitUntil: 'domcontentloaded'
        });

        await page.screenshot({
            path: '/usr/src/app/result.png'
        });
        await browser.close();
    } catch (e) {
        console.error(e);
    }
})();

        

            //...
//👇③クリック後にページが遷移するまで待機
await page.waitForNavigation({
    waitUntil: 'networkidle0'
});
//...

        

            //...
//👇③クリック後にページが遷移するまで待機
await page.waitForSelector('.logout');
//...

        

            //...
//👇③クリック後にページが遷移するまで待機
await page.waitForSelector('.logout', {visible: true});
//...

        

            import puppeteer from 'puppeteer';
import { writeFile } from 'fs';

(async () => {
    try {
        const browser = await puppeteer.launch({
            executablePath: '/usr/bin/chromium-browser',
            args: ['--disable-dev-shm-usage', '--no-sandbox']
        });
        const page = await browser.newPage();
        await page.goto('https://trade.m******.co.jp/mgap/login');

        //👇①input要素のname属性を利用したフォームの入力
        await page.type('input[name="user"]', 'xxxxxxxx');
        await page.type('input[name="password"]', 'xxxxxxxxxxxxx');

        //👇②idがsubmit-btnのボタン要素をクリック
        page.click('#submit-btn');

        //👇③クリック後にページが遷移するまで待機
        await page.waitForNavigation({
            waitUntil: 'domcontentloaded'
        });

        //👇④現在のHTMLをテキストに変換
        let bodyHtml = await page.content();

        //👇⑤正規表現で生のhtmlを好きなように操作
        //例として空白行部分を全て削除し、テキストファイルとして保存
        bodyHtml = bodyHtml.replace(/(^\n|^\s*?\n)/gm, '');
        console.log(bodyHtml);
        writeFile('/usr/src/app/result.txt', bodyHtml, (err) => {
            if (err) { throw err }
            console.log('Done');
        });

        await browser.close();
    } catch (e) {
        console.error(e);
    }
})();