Skip to content

Commit 569d4cd

Browse files
telpiriontelpirion
authored
samples: adds batch prediction, training job for text with SDK use cases (#383)
* samples: adds batch prediction, training job for text SDK use cases
1 parent 1b09988 commit 569d4cd

8 files changed

+386
-0
lines changed

‎samples/model-builder/conftest.py

+13
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,19 @@ def mock_run_automl_image_training_job(mock_image_training_job):
221221
yield mock
222222

223223

224+
@pytest.fixture
225+
def mock_get_automl_text_training_job(mock_text_training_job):
226+
with patch.object(aiplatform, "AutoMLTextTrainingJob") as mock:
227+
mock.return_value = mock_text_training_job
228+
yield mock
229+
230+
231+
@pytest.fixture
232+
def mock_run_automl_text_training_job(mock_text_training_job):
233+
with patch.object(mock_text_training_job, "run") as mock:
234+
yield mock
235+
236+
224237
@pytest.fixture
225238
def mock_get_custom_training_job(mock_custom_training_job):
226239
with patch.object(aiplatform, "CustomTrainingJob") as mock:

‎samples/model-builder/create_batch_prediction_job_sample.py

+6
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@
1717
from google.cloud import aiplatform
1818

1919

20+
# [START aiplatform_sdk_create_batch_prediction_job_text_classification_sample]
21+
# [START aiplatform_sdk_create_batch_prediction_job_text_entity_extraction_sample]
22+
# [START aiplatform_sdk_create_batch_prediction_job_text_sentiment_analysis_sample]
2023
# [START aiplatform_sdk_create_batch_prediction_job_sample]
2124
def create_batch_prediction_job_sample(
2225
project: str,
@@ -46,4 +49,7 @@ def create_batch_prediction_job_sample(
4649
return batch_prediction_job
4750

4851

52+
# [END aiplatform_sdk_create_batch_prediction_job_text_sentiment_analysis_sample]
53+
# [END aiplatform_sdk_create_batch_prediction_job_text_entity_extraction_sample]
54+
# [END aiplatform_sdk_create_batch_prediction_job_text_classification_sample]
4955
# [END aiplatform_sdk_create_batch_prediction_job_sample]

‎samples/model-builder/create_training_pipeline_text_classification_sample.py

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from typing import Optional
16+
17+
from google.cloud import aiplatform
18+
19+
20+
# [START aiplatform_sdk_create_training_pipeline_text_classification_sample]
21+
def create_training_pipeline_text_classification_sample(
22+
project: str,
23+
location: str,
24+
display_name: str,
25+
dataset_id: int,
26+
model_display_name: Optional[str] = None,
27+
multi_label: bool = False,
28+
training_fraction_split: float = 0.8,
29+
validation_fraction_split: float = 0.1,
30+
test_fraction_split: float = 0.1,
31+
budget_milli_node_hours: int = 8000,
32+
disable_early_stopping: bool = False,
33+
sync: bool = True,
34+
):
35+
aiplatform.init(project=project, location=location)
36+
37+
job = aiplatform.AutoMLTextTrainingJob(
38+
display_name=display_name,
39+
prediction_type="classification",
40+
multi_label=multi_label,
41+
)
42+
43+
text_dataset = aiplatform.TextDataset(dataset_id)
44+
45+
model = job.run(
46+
dataset=text_dataset,
47+
model_display_name=model_display_name,
48+
training_fraction_split=training_fraction_split,
49+
validation_fraction_split=validation_fraction_split,
50+
test_fraction_split=test_fraction_split,
51+
budget_milli_node_hours=budget_milli_node_hours,
52+
disable_early_stopping=disable_early_stopping,
53+
sync=sync,
54+
)
55+
56+
model.wait()
57+
58+
print(model.display_name)
59+
print(model.resource_name)
60+
print(model.uri)
61+
return model
62+
63+
64+
# [END aiplatform_sdk_create_training_pipeline_text_classification_sample]

‎samples/model-builder/create_training_pipeline_text_classification_sample_test.py

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
import create_training_pipeline_text_classification_sample
17+
import test_constants as constants
18+
19+
20+
def test_create_training_pipeline_text_classification_sample(
21+
mock_sdk_init,
22+
mock_text_dataset,
23+
mock_get_automl_text_training_job,
24+
mock_run_automl_text_training_job,
25+
mock_get_text_dataset,
26+
):
27+
28+
create_training_pipeline_text_classification_sample.create_training_pipeline_text_classification_sample(
29+
project=constants.PROJECT,
30+
location=constants.LOCATION,
31+
display_name=constants.DISPLAY_NAME,
32+
dataset_id=constants.RESOURCE_ID,
33+
model_display_name=constants.DISPLAY_NAME_2,
34+
training_fraction_split=constants.TRAINING_FRACTION_SPLIT,
35+
validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT,
36+
test_fraction_split=constants.TEST_FRACTION_SPLIT,
37+
budget_milli_node_hours=constants.BUDGET_MILLI_NODE_HOURS_8000,
38+
disable_early_stopping=False,
39+
)
40+
41+
mock_get_text_dataset.assert_called_once_with(constants.RESOURCE_ID)
42+
43+
mock_sdk_init.assert_called_once_with(
44+
project=constants.PROJECT, location=constants.LOCATION
45+
)
46+
mock_get_automl_text_training_job.assert_called_once_with(
47+
display_name=constants.DISPLAY_NAME,
48+
multi_label=False,
49+
prediction_type="classification",
50+
)
51+
mock_run_automl_text_training_job.assert_called_once_with(
52+
dataset=mock_text_dataset,
53+
model_display_name=constants.DISPLAY_NAME_2,
54+
training_fraction_split=constants.TRAINING_FRACTION_SPLIT,
55+
validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT,
56+
test_fraction_split=constants.TEST_FRACTION_SPLIT,
57+
budget_milli_node_hours=constants.BUDGET_MILLI_NODE_HOURS_8000,
58+
disable_early_stopping=False,
59+
sync=True,
60+
)

‎samples/model-builder/create_training_pipeline_text_entity_extraction_sample.py

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from typing import Optional
16+
17+
from google.cloud import aiplatform
18+
19+
20+
# [START aiplatform_sdk_create_training_pipeline_text_entity_extraction_sample]
21+
def create_training_pipeline_text_entity_extraction_sample(
22+
project: str,
23+
location: str,
24+
display_name: str,
25+
dataset_id: int,
26+
model_display_name: Optional[str] = None,
27+
training_fraction_split: float = 0.8,
28+
validation_fraction_split: float = 0.1,
29+
test_fraction_split: float = 0.1,
30+
budget_milli_node_hours: int = 8000,
31+
disable_early_stopping: bool = False,
32+
sync: bool = True,
33+
):
34+
aiplatform.init(project=project, location=location)
35+
36+
job = aiplatform.AutoMLTextTrainingJob(
37+
display_name=display_name, prediction_type="extraction"
38+
)
39+
40+
text_dataset = aiplatform.TextDataset(dataset_id)
41+
42+
model = job.run(
43+
dataset=text_dataset,
44+
model_display_name=model_display_name,
45+
training_fraction_split=training_fraction_split,
46+
validation_fraction_split=validation_fraction_split,
47+
test_fraction_split=test_fraction_split,
48+
budget_milli_node_hours=budget_milli_node_hours,
49+
disable_early_stopping=disable_early_stopping,
50+
sync=sync,
51+
)
52+
53+
model.wait()
54+
55+
print(model.display_name)
56+
print(model.resource_name)
57+
print(model.uri)
58+
return model
59+
60+
61+
# [END aiplatform_sdk_create_training_pipeline_text_entity_extraction_sample]

‎samples/model-builder/create_training_pipeline_text_entity_extraction_sample_test.py

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
import create_training_pipeline_text_entity_extraction_sample
17+
import test_constants as constants
18+
19+
20+
def test_create_training_pipeline_text_clentity_extraction_sample(
21+
mock_sdk_init,
22+
mock_text_dataset,
23+
mock_get_automl_text_training_job,
24+
mock_run_automl_text_training_job,
25+
mock_get_text_dataset,
26+
):
27+
28+
create_training_pipeline_text_entity_extraction_sample.create_training_pipeline_text_entity_extraction_sample(
29+
project=constants.PROJECT,
30+
location=constants.LOCATION,
31+
display_name=constants.DISPLAY_NAME,
32+
dataset_id=constants.RESOURCE_ID,
33+
model_display_name=constants.DISPLAY_NAME_2,
34+
training_fraction_split=constants.TRAINING_FRACTION_SPLIT,
35+
validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT,
36+
test_fraction_split=constants.TEST_FRACTION_SPLIT,
37+
budget_milli_node_hours=constants.BUDGET_MILLI_NODE_HOURS_8000,
38+
disable_early_stopping=False,
39+
)
40+
41+
mock_get_text_dataset.assert_called_once_with(constants.RESOURCE_ID)
42+
43+
mock_sdk_init.assert_called_once_with(
44+
project=constants.PROJECT, location=constants.LOCATION
45+
)
46+
mock_get_automl_text_training_job.assert_called_once_with(
47+
display_name=constants.DISPLAY_NAME, prediction_type="extraction"
48+
)
49+
mock_run_automl_text_training_job.assert_called_once_with(
50+
dataset=mock_text_dataset,
51+
model_display_name=constants.DISPLAY_NAME_2,
52+
training_fraction_split=constants.TRAINING_FRACTION_SPLIT,
53+
validation_fraction_split=constants.VALIDATION_FRACTION_SPLIT,
54+
test_fraction_split=constants.TEST_FRACTION_SPLIT,
55+
budget_milli_node_hours=constants.BUDGET_MILLI_NODE_HOURS_8000,
56+
disable_early_stopping=False,
57+
sync=True,
58+
)

‎samples/model-builder/create_training_pipeline_text_sentiment_analysis_sample.py

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright 2021 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from typing import Optional
16+
17+
from google.cloud import aiplatform
18+
19+
20+
# [START aiplatform_sdk_create_training_pipeline_text_sentiment_analysis_sample]
21+
def create_training_pipeline_text_sentiment_analysis_sample(
22+
project: str,
23+
location: str,
24+
display_name: str,
25+
dataset_id: int,
26+
model_display_name: Optional[str] = None,
27+
sentiment_max: int = 10,
28+
training_fraction_split: float = 0.8,
29+
validation_fraction_split: float = 0.1,
30+
test_fraction_split: float = 0.1,
31+
budget_milli_node_hours: int = 8000,
32+
disable_early_stopping: bool = False,
33+
sync: bool = True,
34+
):
35+
aiplatform.init(project=project, location=location)
36+
37+
job = aiplatform.AutoMLTextTrainingJob(
38+
display_name=display_name,
39+
prediction_type="sentiment",
40+
sentiment_max=sentiment_max,
41+
)
42+
43+
text_dataset = aiplatform.TextDataset(dataset_id)
44+
45+
model = job.run(
46+
dataset=text_dataset,
47+
model_display_name=model_display_name,
48+
training_fraction_split=training_fraction_split,
49+
validation_fraction_split=validation_fraction_split,
50+
test_fraction_split=test_fraction_split,
51+
budget_milli_node_hours=budget_milli_node_hours,
52+
disable_early_stopping=disable_early_stopping,
53+
sync=sync,
54+
)
55+
56+
model.wait()
57+
58+
print(model.display_name)
59+
print(model.resource_name)
60+
print(model.uri)
61+
return model
62+
63+
64+
# [END aiplatform_sdk_create_training_pipeline_text_sentiment_analysis_sample]

0 commit comments

Comments
 (0)