This model has the following steps:
- extract data from BigQuery, transform it and write the transformed data to Cloud Storage.
- Train a TensorFlow Estimator API model and do hyperparameter tuning of the model
- Once the best learning rate, batch size, etc. are determined, train the model longer and on more data using those parameters
- Deploy the trained model to Cloud ML Engine.
preprocess = dsl.ContainerOp(
name='preprocess',
image='gcr.io/cloud-training-demos/babyweight-pipeline-bqtocsv:latest',
arguments=[
'--project', project,
'--mode', 'cloud',
'--bucket', bucket
],
file_outputs={'bucket': '/output.txt'}
)
hparam_train = dsl.ContainerOp(
name='hypertrain',
image='gcr.io/cloud-training-demos/babyweight-pipeline-hypertrain:latest',
arguments=[
preprocess.outputs['bucket']
],
file_outputs={'jobname': '/output.txt'}
)
train_tuned = dsl.ContainerOp(
name='traintuned',
image='gcr.io/cloud-training-demos/babyweight-pipeline-traintuned-trainer:latest',
arguments=[
hparam_train.outputs['jobname'],
bucket
],
file_outputs={'train': '/output.txt'}
)
train_tuned.set_memory_request('2G')
train_tuned.set_cpu_request('1')
deploy_cmle = dsl.ContainerOp(
name='deploycmle',
image='gcr.io/cloud-training-demos/babyweight-pipeline-deploycmle:latest',
arguments=[
train_tuned.outputs['train'], # modeldir
'babyweight',
'mlp'
],
file_outputs={
'model': '/model.txt',
'version': '/version.txt'
}
)注意到kubeflow采用文件的方式来交互不同component
Deploying the notebook as a component
打一个镜像
import kfp.components as comp
import kfp.dsl as dsl
# a single-op pipeline that runs the flights pipeline on the pod
@dsl.pipeline(
name='FlightsPipeline',
description='Trains, deploys flights model'
)
def flights_pipeline(
inputnb=dsl.PipelineParam('inputnb'),
outputnb=dsl.PipelineParam('outputnb'),
params=dsl.PipelineParam('params')
):
notebookop = dsl.ContainerOp(
name='flightsmodel',
image='gcr.io/cloud-training-demos/submitnotebook:latest',
arguments=[
inputnb,
outputnb,
params
]
)