Skip to content

Commit 47c3285

Browse files
authored
chore: Script to inspect and clean up stale GCFs (#331)
Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/python-bigquery-dataframes/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes internal issue 319307783 🦕 ### Usage: ```bash $ python scripts/manage_cloud_functions.py --help usage: manage_cloud_functions.py [-h] -p PROJECT_ID [-r REGIONS] {summary,cleanup} ... Manage cloud functions created to serve bigframes remote functions. options: -h, --help show this help message and exit -p PROJECT_ID, --project-id PROJECT_ID GCP project-id. -r REGIONS, --regions REGIONS Cloud functions region(s). If multiple regions, Specify comma separated (e.g. region1,region2) subcommands: {summary,cleanup} summary BigFrames cloud functions summary. cleanup BigFrames cloud functions clean up. $ python scripts/manage_cloud_functions.py summary --help usage: manage_cloud_functions.py summary [-h] Show the bigframes cloud functions summary. options: -h, --help show this help message and exit $ python scripts/manage_cloud_functions.py cleanup --help usage: manage_cloud_functions.py cleanup [-h] [-n NUMBER] Delete the stale bigframes cloud functions. options: -h, --help show this help message and exit -n NUMBER, --number NUMBER Number of stale (more than a day old) cloud functions to clean up. (venv) shobs@shobs-ct-3:~/code/bigframes1$ ``` ### Example: ```bash $ python scripts/manage_cloud_functions.py -p bigframes-dev summary us-central1: Total=1412, Recent=86, OlderThanADay=1326 europe-west4: Total=270, Recent=24, OlderThanADay=246 southamerica-west1: Total=269, Recent=23, OlderThanADay=246 europe-west1: Total=262, Recent=23, OlderThanADay=239 asia-southeast1: Total=260, Recent=18, OlderThanADay=242 us-east1: Total=1, Recent=0, OlderThanADay=1 $ python scripts/manage_cloud_functions.py -p bigframes-dev -r us-central1,europe-west4 summary us-central1: Total=1412, Recent=85, OlderThanADay=1327 europe-west4: Total=270, Recent=24, OlderThanADay=246 $ python scripts/manage_cloud_functions.py -p bigframes-dev -r us-central1,europe-west4 cleanup -n 2 [us-central1]: deleted [1] projects/bigframes-dev/locations/us-central1/functions/bigframes-597cc02ef5ce0525e4f51697b5a83b6c-3pfpu6gu last updated on 2024-01-08 21:47:58.503628+00:00 [us-central1]: deleted [2] projects/bigframes-dev/locations/us-central1/functions/bigframes-68f796a13666bb3bfe354dd1adaeef71 last updated on 2024-01-09 21:52:49.620259+00:00 [europe-west4]: deleted [1] projects/bigframes-dev/locations/europe-west4/functions/bigframes-558d0ca6649537a9e45896faf08b0a7a last updated on 2024-01-12 21:15:04.379198+00:00 [europe-west4]: deleted [2] projects/bigframes-dev/locations/europe-west4/functions/bigframes-4b7705561ec336ed80722a8e6e56ac41 last updated on 2024-01-08 05:34:59.331828+00:00 $ python scripts/manage_cloud_functions.py -p bigframes-dev -r us-central1,europe-west4 summary us-central1: Total=1410, Recent=85, OlderThanADay=1325 europe-west4: Total=269, Recent=25, OlderThanADay=244 ```
1 parent 75dc9e6 commit 47c3285

File tree

1 file changed

+195
-0
lines changed

1 file changed

+195
-0
lines changed

scripts/manage_cloud_functions.py

+195
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# Copyright 2023 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import argparse
16+
from datetime import datetime
17+
import sys
18+
import time
19+
20+
import google.api_core.exceptions
21+
from google.cloud import functions_v2
22+
23+
GCF_REGIONS_ALL = [
24+
"asia-east1",
25+
"asia-east2",
26+
"asia-northeast1",
27+
"asia-northeast2",
28+
"europe-north1",
29+
"europe-southwest1",
30+
"europe-west1",
31+
"europe-west2",
32+
"europe-west4",
33+
"europe-west8",
34+
"europe-west9",
35+
"us-central1",
36+
"us-east1",
37+
"us-east4",
38+
"us-east5",
39+
"us-south1",
40+
"us-west1",
41+
"asia-east2",
42+
"asia-northeast3",
43+
"asia-southeast1",
44+
"asia-southeast2",
45+
"asia-south1",
46+
"asia-south2",
47+
"australia-southeast1",
48+
"australia-southeast2",
49+
"europe-central2",
50+
"europe-west2",
51+
"europe-west3",
52+
"europe-west6",
53+
"northamerica-northeast1",
54+
"northamerica-northeast2",
55+
"southamerica-east1",
56+
"southamerica-west1",
57+
"us-west2",
58+
"us-west3",
59+
"us-west4",
60+
]
61+
62+
GCF_CLIENT = functions_v2.FunctionServiceClient()
63+
64+
65+
def get_bigframes_functions(project, region):
66+
parent = f"projects/{args.project_id}/locations/{region}"
67+
functions = GCF_CLIENT.list_functions(
68+
functions_v2.ListFunctionsRequest(parent=parent)
69+
)
70+
# Filter bigframes created functions
71+
functions = [
72+
function
73+
for function in functions
74+
if function.name.startswith(
75+
f"projects/{args.project_id}/locations/{region}/functions/bigframes-"
76+
)
77+
]
78+
79+
return functions
80+
81+
82+
def summarize_gcfs(args):
83+
"""Summarize number of bigframes cloud functions in various regions."""
84+
85+
region_counts = {}
86+
for region in args.regions:
87+
functions = get_bigframes_functions(args.project_id, region)
88+
functions_count = len(functions)
89+
90+
# Exclude reporting regions with 0 bigframes GCFs
91+
if functions_count == 0:
92+
continue
93+
94+
# Count how many GCFs are newer than a day
95+
recent = 0
96+
for f in functions:
97+
age = datetime.now() - datetime.fromtimestamp(f.update_time.timestamp())
98+
if age.days <= 0:
99+
recent += 1
100+
101+
region_counts[region] = (functions_count, recent)
102+
103+
for item in sorted(
104+
region_counts.items(), key=lambda item: item[1][0], reverse=True
105+
):
106+
region = item[0]
107+
count, recent = item[1]
108+
print(
109+
"{}: Total={}, Recent={}, OlderThanADay={}".format(
110+
region, count, recent, count - recent
111+
)
112+
)
113+
114+
115+
def cleanup_gcfs(args):
116+
"""Clean-up bigframes cloud functions in the given regions."""
117+
max_delete_per_region = args.number
118+
119+
for region in args.regions:
120+
functions = get_bigframes_functions(args.project_id, region)
121+
count = 0
122+
for f in functions:
123+
age = datetime.now() - datetime.fromtimestamp(f.update_time.timestamp())
124+
if age.days > 0:
125+
try:
126+
count += 1
127+
GCF_CLIENT.delete_function(name=f.name)
128+
print(
129+
f"[{region}]: deleted [{count}] {f.name} last updated on {f.update_time}"
130+
)
131+
if count >= max_delete_per_region:
132+
break
133+
# Mostly there is a 60 mutations per minute quota, we want to use 10% of
134+
# that for this clean-up, i.e. 6 mutations per minute. So wait for
135+
# 60/6 = 10 seconds
136+
time.sleep(10)
137+
except google.api_core.exceptions.ResourceExhausted:
138+
# Stop deleting in this region for now
139+
print(
140+
f"Cannot delete any more functions in region {region} due to quota exhaustion. Please try again later."
141+
)
142+
break
143+
144+
145+
def list_str(values):
146+
return [val for val in values.split(",") if val]
147+
148+
149+
if __name__ == "__main__":
150+
parser = argparse.ArgumentParser(
151+
description="Manage cloud functions created to serve bigframes remote functions."
152+
)
153+
parser.add_argument(
154+
"-p",
155+
"--project-id",
156+
type=str,
157+
required=True,
158+
action="store",
159+
help="GCP project-id.",
160+
)
161+
parser.add_argument(
162+
"-r",
163+
"--regions",
164+
type=list_str,
165+
required=False,
166+
default=GCF_REGIONS_ALL,
167+
action="store",
168+
help="Cloud functions region(s). If multiple regions, Specify comma separated (e.g. region1,region2)",
169+
)
170+
171+
subparsers = parser.add_subparsers(title="subcommands", required=True)
172+
parser_summary = subparsers.add_parser(
173+
"summary",
174+
help="BigFrames cloud functions summary.",
175+
description="Show the bigframes cloud functions summary.",
176+
)
177+
parser_summary.set_defaults(func=summarize_gcfs)
178+
parser_cleanup = subparsers.add_parser(
179+
"cleanup",
180+
help="BigFrames cloud functions clean up.",
181+
description="Delete the stale bigframes cloud functions.",
182+
)
183+
parser_cleanup.add_argument(
184+
"-n",
185+
"--number",
186+
type=int,
187+
required=False,
188+
default=100,
189+
action="store",
190+
help="Number of stale (more than a day old) cloud functions to clean up.",
191+
)
192+
parser_cleanup.set_defaults(func=cleanup_gcfs)
193+
194+
args = parser.parse_args(sys.argv[1:])
195+
args.func(args)

0 commit comments

Comments
 (0)