Feat: Onboard Celeba dataset (#420)

Author: Naveen130

datasets/celeba/infra/provider.tf

@@ -0,0 +1,28 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+provider "google" {
+  project                     = var.project_id
+  impersonate_service_account = var.impersonating_acct
+  region                      = var.region
+}
+
+data "google_client_openid_userinfo" "me" {}
+
+output "impersonating-account" {
+  value = data.google_client_openid_userinfo.me.email
+}

datasets/celeba/infra/variables.tf

@@ -0,0 +1,26 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+variable "project_id" {}
+variable "bucket_name_prefix" {}
+variable "impersonating_acct" {}
+variable "region" {}
+variable "env" {}
+variable "iam_policies" {
+  default = {}
+}
+

datasets/celeba/pipelines/celeba/celeba_dag.py

@@ -0,0 +1,44 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from airflow import DAG
+from airflow.providers.google.cloud.transfers import gcs_to_gcs
+
+default_args = {
+    "owner": "Google",
+    "depends_on_past": False,
+    "start_date": "2022-07-20",
+}
+
+
+with DAG(
+    dag_id="celeba.celeba",
+    default_args=default_args,
+    max_active_runs=1,
+    schedule_interval="@once",
+    catchup=False,
+    default_view="graph",
+) as dag:
+
+    # Transfer data from source to destination in GCS
+    GCStoGCS_transfer = gcs_to_gcs.GCSToGCSOperator(
+        task_id="GCStoGCS_transfer",
+        source_bucket="{{ var.json.celeba.source_bucket }}",
+        source_object="{{ var.json.celeba.source_object }}",
+        destination_bucket="{{ var.json.celeba.destination_bucket }}",
+        destination_object="{{ var.json.celeba.destination_object }}",
+    )
+
+    GCStoGCS_transfer

datasets/celeba/pipelines/celeba/pipeline.yaml

@@ -0,0 +1,42 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+resources: ~
+
+dag:
+  airflow_version: 2
+  initialize:
+    dag_id: celeba
+    default_args:
+      owner: "Google"
+      depends_on_past: False
+      start_date: "2022-07-20"
+    max_active_runs: 1
+    schedule_interval: "@once"
+    catchup: False
+    default_view: graph
+
+  tasks:
+    - operator: "GoogleCloudStorageToGoogleCloudStorageOperator"
+      description: "Transfer data from source to destination in GCS"
+      args:
+        task_id: "GCStoGCS_transfer"
+        source_bucket: "{{ var.json.celeba.source_bucket }}"
+        source_object: "{{ var.json.celeba.source_object }}"
+        destination_bucket: "{{ var.json.celeba.destination_bucket }}"
+        destination_object: "{{ var.json.celeba.destination_object }}"
+
+  graph_paths:
+    - "GCStoGCS_transfer"

datasets/celeba/pipelines/dataset.yaml

@@ -0,0 +1,23 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dataset:
+  name: celeba
+  friendly_name: Celebrity Attributes
+  description: Dataset of images of the facial attributes of various celebrities.
+  dataset_sources: ~
+  terms_of_use: ~
+
+
+resources: ~