Rate-limit remote image checks

package.json

@@ -32,6 +32,7 @@
     "parse-passwd": "^1.0.0",
     "prettier": "^2.3.1",
     "regenerator-runtime": "^0.13.7",
+    "semaphore": "^1.1.0",
     "strip-ansi": "^6.0.0",
     "style-loader": "^2.0.0",
     "uuid": "^8.3.2",
@@ -41,5 +42,9 @@
     "xterm": "^4.9.0",
     "xterm-addon-fit": "^0.4.0",
     "yaml": "^1.10.0"
-  }
+  },
+  "$comments": [
+    "limiter version pinned due to https://github.com/jhurliman/node-rate-limiter/issues/80",
+    "monaco-languageclient, monaco-editor, vscode-languageserver-protocol pinned because their APIs changed a bunch and Riju hasn't been updated yet"
+  ]
 }

tools/depgraph.js

@@ -23,6 +23,22 @@ import {
 import { getBaseImages, hashDockerfile } from "./hash-dockerfile.js";
 import { runCommand } from "./util.js";
 
+const CONCURRENCY = 1;
+
+async function allPromises(callables, { concurrency }) {
+  const queue = new PQueue({ concurrency: concurrency });
+  const results = [];
+  for (const callable of callables) {
+    queue.add(async () => {
+      console.log("START");
+      results.push(await callable());
+      console.log("END");
+    });
+  }
+  await queue.onIdle();
+  return results;
+}
+
 function getS3Bucket() {
   if (!process.env.S3_BUCKET) {
     throw new Error(`unset environment variable: \$S3_BUCKET`);

tools/docker-util.js

@@ -1,5 +1,7 @@
 import process from "process";
 
+import semaphore from "semaphore";
+
 import { runCommand } from "./util.js";
 
 // Return the digest of a local image. This is the actual image
@@ -41,33 +43,40 @@ export async function getRemoteRepositoryTags(repo) {
   ).Tags;
 }
 
+const remoteImageRateLimiter = semaphore(16);
+
 // Return the value of a label on a Docker image that is on a remote
 // registry. If the image or label doesn't exist, return null. You
 // have to pass in a list of tags on the remote repository (see
 // getRemoteRepositoryTags) so that we can distinguish between missing
 // images and network errors.
 export async function getRemoteImageLabel(image, label, tags) {
-  const [repo, tag] = image.split(":");
-  let output;
+  await new Promise((resolve) => remoteImageRateLimiter.take(resolve));
   try {
-    output = (
-      await runCommand(`skopeo inspect docker://${image}`, {
-        getStdout: true,
-      })
-    ).stdout;
-  } catch (err) {
-    if (tags.includes(tag)) {
-      // Tag exists, something unexpected must have gone wrong when
-      // running skopeo inspect.
-      throw err;
-    } else {
-      // Tag does not exist, that must be why skopeo inspect didn't
-      // work.
-      return null;
+    const [_repo, tag] = image.split(":");
+    let output;
+    try {
+      output = (
+        await runCommand(`skopeo inspect docker://${image}`, {
+          getStdout: true,
+        })
+      ).stdout;
+    } catch (err) {
+      if (tags.includes(tag)) {
+        // Tag exists, something unexpected must have gone wrong when
+        // running skopeo inspect.
+        throw err;
+      } else {
+        // Tag does not exist, that must be why skopeo inspect didn't
+        // work.
+        return null;
+      }
     }
+    const labels = JSON.parse(output).Labels;
+    return (labels && labels[label]) || null;
+  } finally {
+    remoteImageRateLimiter.leave();
   }
-  const labels = JSON.parse(output).Labels;
-  return (labels && labels[label]) || null;
 }
 
 // Return the value of $DOCKER_REPO, throwing an error if it's not set

yarn.lock

@@ -3927,6 +3927,11 @@ schema-utils@^3.0.0:
     ajv "^6.12.5"
     ajv-keywords "^3.5.2"
 
+semaphore@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/semaphore/-/semaphore-1.1.0.tgz#aaad8b86b20fe8e9b32b16dc2ee682a8cd26a8aa"
+  integrity sha512-O4OZEaNtkMd/K0i6js9SL+gqy0ZCBMgUvlSqHKi4IBdjhe7wB8pwztUk1BbZ1fmrvpwFrPbHzqd2w5pTcJH6LA==
+
 semver@7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e"