【tensorflowjs入門】Nexeでシェルスクリプトからデータ解析できるツールを作成する

            
            $ yarn add @tensorflow/tfjs
        

            
            import * as tf from '@tensorflow/tfjs';

(() => {
    const t = tf.tensor([-2, 1, 0, 5]);
    const o = tf.layers.activation({activation: 'relu'}).apply(t);
    (o as any).print();
})();
        

            
            $ tsc
$ nexe dist/index.js --target=linux-x64-14.15.3 -o nexe-app
        

            
            $ ./nexe-app
============================
Hi there 👋. Looks like you are running TensorFlow.js in Node.js. To speed things up dramatically, install our node backend, which binds to TensorFlow C++, by running npm i @tensorflow/tfjs-node, or npm i @tensorflow/tfjs-node-gpu if you have CUDA. Then call require('@tensorflow/tfjs-node'); (-gpu suffix for CUDA) at the start of your program. Visit https://github.com/tensorflow/tfjs-node for more details.
============================
Tensor
    [0, 1, 0, 5]
        

            
            $ mkdir ./tmp
$ cd tmp && wget https://archive.ics.uci.edu/ml/machine-learning-databases/auto-mpg/auto-mpg.data
$ cat auto-mpg.data | sed -r '/\?/d' | sed -r 's/\t/,/' | sed -r 's/ {2,}/,/g' > output.csv
        

            
            $ cat output.csv | sort -k5 -t, | awk -F, '{print $5","$1}'
1613.,35.0
1649.,31.0
1755.,39.1
#...中略
5140.,13.0
        

            
            $ cat input.csv | ./nexe-app
#解析結果が表示される
        

            
            (() => {
    console.log(process.argv[0]);
    console.log(process.argv[1]);
    console.log(process.argv[2]);
    console.log(process.argv[3]);
})();
        

            
            $ ./nexe-app '太郎, 花子' '次郎, 良子'
/[現在の作業ディレクトリ]/nexe-app
/[現在の作業ディレクトリ]/dist/index.js
太郎, 花子
次郎, 良子
        

            
            #!/bin/bash

if [ -p /dev/stdin ]; then
    _str=$(cat -)
else
    _str=$@
fi

./nexe-app "${_str}"
        

            
            $ chmod +x nexe-pipe.sh

$./nexe-pipe.sh '太郎, 花子'
/[現在の作業ディレクトリ]/nexe-app
/[現在の作業ディレクトリ]/dist/index.js
太郎, 花子 次郎, 良子
undefined

$ echo '太郎, 花子' | ./nexe-pipe.sh
/home/taconocat/tacochart/tacochart_viewer/tca/hurst_analysis/hurst-app
/home/taconocat/tacochart/tacochart_viewer/tca/hurst_analysis/dist/index.js
太郎, 花子
undefined
        

            
            function csv2Array(str: string): number[][] {
    const csvData = [];
    const lines = str.split('\n');
    for (let i = 0; i < lines.length; ++i) {
        if (lines[i] == '') { continue; }
        const cells = lines[i].split(',');
        csvData.push(cells.map(v => parseFloat(v)));
    }
    return csvData;
}

(() => {
    const rawCSV = process.argv[2];
    if (rawCSV) console.dir(csv2Array(rawCSV));
})();
        

            
            $ cat tmp/output.csv | sort -k5 -t, | awk -F, '{print $5","$1}' | ./nexe-pipe.sh
[
  [ 1613, 35 ],   [ 1649, 31 ],   [ 1755, 39.1 ], [ 1760, 35.1 ], [ 1773, 31 ],
  [ 1795, 33 ],   [ 1795, 33 ],   [ 1800, 36.1 ], [ 1800, 36.1 ], [ 1825, 29.5 ],
#...中略
  [ 2220, 23 ],   [ 2220, 25 ],   [ 2223, 25 ],   [ 2226, 21 ],   [ 2228, 25 ],
  ... 292 more items
]
        

            
            import * as tf from '@tensorflow/tfjs';

function csv2Array(str: string): number[][] {
    const csvData = [];
    const lines = str.split('\n');
    for (let i = 0; i < lines.length; ++i) {
        if (lines[i] == '') { continue; }
        const cells = lines[i].split(',');
        csvData.push(cells.map(v => parseFloat(v)));
    }
    return csvData;
}

async function linearRegression(rawData: number[][]) {
    const model = tf.sequential();

    //👇入力層兼フィッテング解析関数
    model.add(tf.layers.dense({inputShape: [1], units: 1, useBias: true, activation: 'linear'}));

    console.log('Training started.');

    const originalData = [];
    for (const row in rawData) {
        originalData.push({x: rawData[row][0], y: rawData[row][1]});
    };

    //👇テンソルに変換・データの正規化
    const {inputs, labels, inputMax, inputMin, labelMax, labelMin} = convertToTensor(originalData);

    //👇モデルをトレーニング(=線型回帰解析)
    await trainModel(model, inputs, labels);

    //👇解析前に正規化・標準化していたデータを考慮して、元のXY空間に復元
    const xmax = inputMax.dataSync()[0];
    const xmin = inputMin.dataSync()[0];
    const ymax = labelMax.dataSync()[0];
    const ymin = labelMin.dataSync()[0];
    let prop = model.layers[0].getWeights()[0].dataSync()[0];
    let bias = model.layers[0].getWeights()[1].dataSync()[0];
    bias = ymin + (ymax - ymin) * (bias - prop * xmin / (xmax - xmin));
    prop = prop * (ymax - ymin) / (xmax - xmin);

    console.log(`推定曲線: y = ${prop} x + ${bias}`);

    console.log('Fitting has done.');
}

//👇学習データをTensor型に変換する関数
function convertToTensor(data: Array<{x:number, y:number}>) {
    return tf.tidy(() => {
        //👇学習データをシャッフル
        tf.util.shuffle(data);

        //👇xyデータ配列をNx1テンソルに変換
        const inputs = data.map(d => d.x)
        const labels = data.map(d => d.y);
        const inputTensor = tf.tensor2d(inputs, [inputs.length, 1]);
        const labelTensor = tf.tensor2d(labels, [labels.length, 1]);

        //👇入力データの正規化
        const inputMax = inputTensor.max();
        const inputMin = inputTensor.min();
        const labelMax = labelTensor.max();
        const labelMin = labelTensor.min();
        const normalizedInputs = inputTensor.sub(inputMin).div(inputMax.sub(inputMin));
        const normalizedLabels = labelTensor.sub(labelMin).div(labelMax.sub(labelMin));

        return {
            inputs: normalizedInputs,
            labels: normalizedLabels,
            inputMax,
            inputMin,
            labelMax,
            labelMin
        }
    });
}

//👇モデルの学習を行う関数(=線型回帰解析本体)
async function trainModel(model: tf.Sequential, inputs: tf.Tensor, labels: tf.Tensor) {
    //👇モデルをコンパイル => 学習方法を指定
    model.compile({
        optimizer: tf.train.adam(),
        loss: tf.losses.meanSquaredError,
        metrics: ['mse'],
    });

    //👇バッチサイズ(16~32くらいで良い)
    const batchSize = 32;
    //👇エポック数
    const epochs = 500;
    //👇エポック回数の学習を実行する
    return await model.fit(inputs, labels, {
        batchSize,
        epochs,
        shuffle: true,
        callbacks: {
            onEpochEnd: async(epoch, logs) => {
                //👇繰り返し回数と損失をコンソール出力
                console.log(`Epoch#${epoch} : Loss(${logs.loss}) : mse(${logs.mse})`);
            }
        }
    });
}

(async () => {
    const rawCSV = process.argv[2];
    if (rawCSV) {
        const csvData = csv2Array(rawCSV);
        await linearRegression(csvData);
    }
})();
        

            
            $ cat tmp/output.csv | sort -k5 -t, | awk -F, '{print $5","$1}' | ./nexe-pipe.sh

============================
Hi there 👋. Looks like you are running TensorFlow.js in Node.js. To speed things up dramatically, install our node backend, which binds to TensorFlow C++, by running npm i @tensorflow/tfjs-node, or npm i @tensorflow/tfjs-node-gpu if you have CUDA. Then call require('@tensorflow/tfjs-node'); (-gpu suffix for CUDA) at the start of your program. Visit https://github.com/tensorflow/tfjs-node for more details.
============================
Training started.
Epoch#0 : Loss(0.4150969684123993) : mse(0.4150969684123993)
Epoch#1 : Loss(0.40245047211647034) : mse(0.40245047211647034)
Epoch#2 : Loss(0.3915448486804962) : mse(0.3915448486804962)
#...中略
Epoch#498 : Loss(0.013235836289823055) : mse(0.013235836289823055)
Epoch#499 : Loss(0.013220091350376606) : mse(0.013220091350376606)
推定曲線: y = -0.0076351825945764045 x + 46.188474321707034
Fitting has done.