Storage Emulator "Fast-Follow" Bug Fixes

CHANGELOG.md

@@ -2,3 +2,6 @@
 - Adds the deployed version to the output when deploying to Firebase Hosting.
 - Updates the configured runtime for functions generated by `firebase init functions` from Node.js 12 to 14 (#3399)
 - Releases Firestore Emulator v1.12.0: supports clearing data partially.
+- Fixes manually setting download tokens in Storage Emulator. (#3396)
+- Fixes deleting custom metadata in Storage emulator. (#3385)
+- Fixes errors when calling makePublic() with Storage Emulator(#3394)

scripts/storage-emulator-integration/storage.rules

@@ -4,6 +4,10 @@ service firebase.storage {
     match /{allPaths=**} {
       allow read, write: if request.auth != null;
     }
+
+    match /public/{allPaths=**} {
+      allow read;
+    }
   }
 }
 

scripts/storage-emulator-integration/tests.ts

@@ -298,6 +298,52 @@ describe("Storage emulator", () => {
         });
       });
 
+      describe("#makePublic()", () => {
+        it("should no-op", async () => {
+          const destination = "a/b";
+          await testBucket.upload(smallFilePath, { destination });
+          const [aclMetadata] = await testBucket.file(destination).makePublic();
+
+          const generation = aclMetadata.generation;
+          delete aclMetadata.generation;
+
+          expect(aclMetadata).to.deep.equal({
+            kind: "storage#objectAccessControl",
+            object: destination,
+            id: `${testBucket.name}/${destination}/${generation}/allUsers`,
+            selfLink: `${STORAGE_EMULATOR_HOST}/storage/v1/b/${
+              testBucket.name
+            }/o/${encodeURIComponent(destination)}/acl/allUsers`,
+            bucket: testBucket.name,
+            entity: "allUsers",
+            role: "READER",
+            etag: "someEtag",
+          });
+        });
+
+        it("should not interfere with downloading of bytes via public URL", async () => {
+          const destination = "a/b";
+          await testBucket.upload(smallFilePath, { destination });
+          await testBucket.file(destination).makePublic();
+
+          const publicLink = `${STORAGE_EMULATOR_HOST}/${testBucket.name}/${destination}`;
+
+          const requestClient = TEST_CONFIG.useProductionServers ? https : http;
+          await new Promise((resolve, reject) => {
+            requestClient.get(publicLink, {}, (response) => {
+              const data: any = [];
+              response
+                .on("data", (chunk) => data.push(chunk))
+                .on("end", () => {
+                  expect(Buffer.concat(data).length).to.equal(SMALL_FILE_SIZE);
+                })
+                .on("close", resolve)
+                .on("error", reject);
+            });
+          });
+        });
+      });
+
       describe("#getMetadata()", () => {
         it("should throw on non-existing file", async () => {
           let err: any;
@@ -366,6 +412,38 @@ describe("Storage emulator", () => {
             });
           });
         });
+
+        it("should handle firebaseStorageDownloadTokens", async () => {
+          const destination = "public/small_file";
+          await testBucket.upload(smallFilePath, {
+            destination,
+            metadata: {},
+          });
+
+          const cloudFile = testBucket.file(destination);
+          const md = {
+            metadata: {
+              firebaseStorageDownloadTokens: "myFirstToken,mySecondToken",
+            },
+          };
+
+          await cloudFile.setMetadata(md);
+
+          // Check that the tokens are saved in Firebase metadata
+          await supertest(STORAGE_EMULATOR_HOST)
+            .get(`/v0/b/${testBucket.name}/o/${encodeURIComponent(destination)}`)
+            .expect(200)
+            .then((res) => {
+              const firebaseMd = res.body;
+              expect(firebaseMd.downloadTokens).to.equal(md.metadata.firebaseStorageDownloadTokens);
+            });
+
+          // Check that the tokens are saved in Cloud metadata
+          const [metadata] = await cloudFile.getMetadata();
+          expect(metadata.metadata.firebaseStorageDownloadTokens).to.deep.equal(
+            md.metadata.firebaseStorageDownloadTokens
+          );
+        });
       });
 
       describe("#setMetadata()", () => {
@@ -987,22 +1065,50 @@ describe("Storage emulator", () => {
         });
       });
 
-      it("#setMetadata()", async () => {
-        const metadata = await page.evaluate((filename) => {
-          return firebase
-            .storage()
-            .ref(filename)
-            .updateMetadata({
+      describe("#setMetadata()", () => {
+        it("should allow for custom metadata to be set", async () => {
+          const metadata = await page.evaluate((filename) => {
+            return firebase
+              .storage()
+              .ref(filename)
+              .updateMetadata({
+                customMetadata: {
+                  is_over: "9000",
+                },
+              })
+              .then(() => {
+                return firebase.storage().ref(filename).getMetadata();
+              });
+          }, filename);
+
+          expect(metadata.customMetadata.is_over).to.equal("9000");
+        });
+
+        it("should allow deletion of custom metadata by setting to null", async () => {
+          const setMetadata = await page.evaluate((filename) => {
+            const storageReference = firebase.storage().ref(filename);
+            return storageReference.updateMetadata({
+              contentType: "text/plain",
               customMetadata: {
-                is_over: "9000",
+                removeMe: "please",
               },
-            })
-            .then(() => {
-              return firebase.storage().ref(filename).getMetadata();
             });
-        }, filename);
+          }, filename);
 
-        expect(metadata.customMetadata.is_over).to.equal("9000");
+          expect(setMetadata.customMetadata.removeMe).to.equal("please");
+
+          const nulledMetadata = await page.evaluate((filename) => {
+            const storageReference = firebase.storage().ref(filename);
+            return storageReference.updateMetadata({
+              contentType: "text/plain",
+              customMetadata: {
+                removeMe: null as any,
+              },
+            });
+          }, filename);
+
+          expect(nulledMetadata.customMetadata.removeMe).to.equal(undefined);
+        });
       });
 
       it("#delete()", async () => {

src/emulator/storage/apis/gcloud.ts

@@ -1,9 +1,16 @@
 import { Router } from "express";
 import { gunzipSync } from "zlib";
 import { Emulators } from "../../types";
-import { CloudStorageObjectMetadata } from "../metadata";
+import {
+  CloudStorageObjectAccessControlMetadata,
+  CloudStorageObjectMetadata,
+  StoredFileMetadata,
+} from "../metadata";
 import { EmulatorRegistry } from "../../registry";
 import { StorageEmulator } from "../index";
+import { EmulatorLogger } from "../../emulatorLogger";
+import { StorageLayer } from "../files";
+import type { Request, Response } from "express";
 
 /**
  * @param emulator
@@ -14,19 +21,19 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
   const gcloudStorageAPI = Router();
   const { storageLayer } = emulator;
 
+  // Automatically create a bucket for any route which uses a bucket
+  gcloudStorageAPI.use(/.*\/b\/(.+?)\/.*/, (req, res, next) => {
+    storageLayer.createBucket(req.params[0]);
+    next();
+  });
+
   gcloudStorageAPI.get("/b", (req, res) => {
     res.json({
       kind: "storage#buckets",
       items: storageLayer.listBuckets(),
     });
   });
 
-  // Automatically create a bucket for any route which uses a bucket
-  gcloudStorageAPI.use(/.*\/b\/(.+?)\/.*/, (req, res, next) => {
-    storageLayer.createBucket(req.params[0]);
-    next();
-  });
-
   gcloudStorageAPI.get(
     ["/b/:bucketId/o/:objectId", "/download/storage/v1/b/:bucketId/o/:objectId"],
     (req, res) => {
@@ -38,40 +45,7 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
       }
 
       if (req.query.alt == "media") {
-        let data = storageLayer.getBytes(req.params.bucketId, req.params.objectId);
-        if (!data) {
-          res.sendStatus(404);
-          return;
-        }
-
-        const isGZipped = md.contentEncoding == "gzip";
-        if (isGZipped) {
-          data = gunzipSync(data);
-        }
-
-        res.setHeader("Accept-Ranges", "bytes");
-        res.setHeader("Content-Type", md.contentType);
-        res.setHeader("Content-Disposition", md.contentDisposition);
-        res.setHeader("Content-Encoding", "identity");
-
-        const byteRange = [...(req.header("range") || "").split("bytes="), "", ""];
-
-        const [rangeStart, rangeEnd] = byteRange[1].split("-");
-
-        if (rangeStart) {
-          const range = {
-            start: parseInt(rangeStart),
-            end: rangeEnd ? parseInt(rangeEnd) : data.byteLength,
-          };
-          res.setHeader(
-            "Content-Range",
-            `bytes ${range.start}-${range.end - 1}/${data.byteLength}`
-          );
-          res.status(206).end(data.slice(range.start, range.end));
-        } else {
-          res.end(data);
-        }
-        return;
+        return sendFileBytes(md, storageLayer, req, res);
       }
 
       const outgoingMd = new CloudStorageObjectMetadata(md);
@@ -165,6 +139,39 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
     res.status(200).json(new CloudStorageObjectMetadata(finalizedUpload.file.metadata)).send();
   });
 
+  gcloudStorageAPI.post("/b/:bucketId/o/:objectId/acl", (req, res) => {
+    // TODO(abehaskins) Link to a doc with more info
+    EmulatorLogger.forEmulator(Emulators.STORAGE).log(
+      "WARN_ONCE",
+      "Cloud Storage ACLs are not supported in the Storage Emulator. All related methods will succeed, but have no effect."
+    );
+    const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);
+
+    if (!md) {
+      res.sendStatus(404);
+      return;
+    }
+
+    // We do an empty update to step metageneration forward;
+    md.update({});
+
+    res
+      .json({
+        kind: "storage#objectAccessControl",
+        object: md.name,
+        id: `${req.params.bucketId}/${md.name}/${md.generation}/allUsers`,
+        selfLink: `http://${EmulatorRegistry.getInfo(Emulators.STORAGE)?.host}:${
+          EmulatorRegistry.getInfo(Emulators.STORAGE)?.port
+        }/storage/v1/b/${md.bucket}/o/${encodeURIComponent(md.name)}/acl/allUsers`,
+        bucket: md.bucket,
+        entity: req.body.entity,
+        role: req.body.role,
+        etag: "someEtag",
+        generation: md.generation.toString(),
+      } as CloudStorageObjectAccessControlMetadata)
+      .status(200);
+  });
+
   gcloudStorageAPI.post("/upload/storage/v1/b/:bucketId/o", (req, res) => {
     if (!req.query.name) {
       res.sendStatus(400);
@@ -250,5 +257,65 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
     return;
   });
 
+  gcloudStorageAPI.get("/:bucketId/:objectId(**)", (req, res) => {
+    const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);
+
+    if (!md) {
+      res.sendStatus(404);
+      return;
+    }
+
+    return sendFileBytes(md, storageLayer, req, res);
+  });
+
+  gcloudStorageAPI.all("/**", (req, res) => {
+    if (process.env.STORAGE_EMULATOR_DEBUG) {
+      console.table(req.headers);
+      console.log(req.method, req.url);
+      res.json("endpoint not implemented");
+    } else {
+      res.sendStatus(501);
+    }
+  });
+
   return gcloudStorageAPI;
 }
+
+function sendFileBytes(
+  md: StoredFileMetadata,
+  storageLayer: StorageLayer,
+  req: Request,
+  res: Response
+) {
+  let data = storageLayer.getBytes(req.params.bucketId, req.params.objectId);
+  if (!data) {
+    res.sendStatus(404);
+    return;
+  }
+
+  const isGZipped = md.contentEncoding == "gzip";
+  if (isGZipped) {
+    data = gunzipSync(data);
+  }
+
+  res.setHeader("Accept-Ranges", "bytes");
+  res.setHeader("Content-Type", md.contentType);
+  res.setHeader("Content-Disposition", md.contentDisposition);
+  res.setHeader("Content-Encoding", "identity");
+
+  const byteRange = [...(req.header("range") || "").split("bytes="), "", ""];
+
+  const [rangeStart, rangeEnd] = byteRange[1].split("-");
+
+  if (rangeStart) {
+    const range = {
+      start: parseInt(rangeStart),
+      end: rangeEnd ? parseInt(rangeEnd) : data.byteLength,
+    };
+    res.setHeader("Content-Range", `bytes ${range.start}-${range.end - 1}/${data.byteLength}`);
+    res.status(206).end(data.slice(range.start, range.end));
+  } else {
+    res.end(data);
+  }
+  return;
+}