Clean databases folder on startup (#675)

Cleans orphan databases on startup. This commit also bumps the fs-extra
dependency to get readdir with dirent objects.

Adds the `asyncFilter` to filter arrays asynchronously.

Author: aeisenberg

extensions/ql-vscode/CHANGELOG.md

@@ -7,6 +7,7 @@
 - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
 - Alter structure of the _Test Explorer_ tree. It now follows the structure of the filesystem instead of the QL Packs. [#624](https://github.com/github/vscode-codeql/pull/624)
 - Add more structured output for tests. [#626](https://github.com/github/vscode-codeql/pull/626)
+- Whenever the extension restarts, orphaned databases will be cleaned up. These are databases whose files are located inside of the extension's storage area, but are not imported into the workspace.
 
 ## 1.3.6 - 4 November 2020
 

extensions/ql-vscode/package-lock.json

@@ -197,6 +197,10 @@
           "dark": "media/dark/folder-opened-plus.svg"
         }
       },
+      {
+        "command": "codeQLDatabases.removeOrphanedDatabases",
+        "title": "Delete unused databases"
+      },
       {
         "command": "codeQLDatabases.chooseDatabaseArchive",
         "title": "Choose Database from Archive",
@@ -573,6 +577,10 @@
           "command": "codeQLDatabases.chooseDatabaseArchive",
           "when": "false"
         },
+        {
+          "command": "codeQLDatabases.removeOrphanedDatabases",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.chooseDatabaseInternet",
           "when": "false"
@@ -704,7 +712,7 @@
   "dependencies": {
     "child-process-promise": "^2.2.1",
     "classnames": "~2.2.6",
-    "fs-extra": "^8.1.0",
+    "fs-extra": "^9.0.1",
     "glob-promise": "^3.4.0",
     "js-yaml": "^3.14.0",
     "minimist": "~1.2.5",
@@ -727,15 +735,15 @@
     "@types/chai-as-promised": "~7.1.2",
     "@types/child-process-promise": "^2.2.1",
     "@types/classnames": "~2.2.9",
-    "@types/fs-extra": "^8.0.0",
+    "@types/fs-extra": "^9.0.3",
     "@types/glob": "^7.1.1",
     "@types/google-protobuf": "^3.2.7",
     "@types/gulp": "^4.0.6",
     "@types/gulp-sourcemaps": "0.0.32",
     "@types/js-yaml": "^3.12.5",
     "@types/jszip": "~3.1.6",
     "@types/mocha": "~8.0.3",
-    "@types/node": "^12.0.8",
+    "@types/node": "^12.14.1",
     "@types/node-fetch": "~2.5.2",
     "@types/proxyquire": "~1.3.28",
     "@types/react": "^16.8.17",

extensions/ql-vscode/package.json

@@ -8,7 +8,7 @@ import {
   TreeItem,
   Uri,
   window,
-  env
+  env,
 } from 'vscode';
 import * as fs from 'fs-extra';
 
@@ -18,6 +18,8 @@ import {
   DatabaseItem,
   DatabaseManager,
   getUpgradesDirectories,
+  isLikelyDatabaseRoot,
+  isLikelyDbLanguageFolder,
 } from './databases';
 import {
   commandRunner,
@@ -36,6 +38,7 @@ import {
   promptImportLgtmDatabase,
 } from './databaseFetcher';
 import { CancellationToken } from 'vscode-jsonrpc';
+import { asyncFilter } from './pure/helpers-pure';
 
 type ThemableIconPath = { light: string; dark: string } | string;
 
@@ -229,7 +232,9 @@ export class DatabaseUI extends DisposableObject {
         canSelectMany: true,
       })
     );
+  }
 
+  init() {
     logger.log('Registering database panel commands.');
     this.push(
       commandRunnerWithProgress(
@@ -340,6 +345,12 @@ export class DatabaseUI extends DisposableObject {
         this.handleOpenFolder
       )
     );
+    this.push(
+      commandRunner(
+        'codeQLDatabases.removeOrphanedDatabases',
+        this.handleRemoveOrphanedDatabases
+      )
+    );
   }
 
   private handleMakeCurrentDatabase = async (
@@ -360,6 +371,53 @@ export class DatabaseUI extends DisposableObject {
     }
   };
 
+  handleRemoveOrphanedDatabases = async (): Promise<void> => {
+    logger.log('Removing orphaned databases from workspace storage.');
+    let dbDirs =
+      // read directory
+      (await fs.readdir(this.storagePath, { withFileTypes: true }))
+        // remove non-directories
+        .filter(dirent => dirent.isDirectory())
+        // get the full path
+        .map(dirent => path.join(this.storagePath, dirent.name))
+        // remove databases still in workspace
+        .filter(dbDir => {
+          const dbUri = Uri.file(dbDir);
+          return this.databaseManager.databaseItems.every(item => item.databaseUri.fsPath !== dbUri.fsPath);
+        });
+
+    // remove non-databases
+    dbDirs = await asyncFilter(dbDirs, isLikelyDatabaseRoot);
+
+    if (!dbDirs.length) {
+      logger.log('No orphaned databases found.');
+      return;
+    }
+
+    // delete
+    const failures = [] as string[];
+    await Promise.all(
+      dbDirs.map(async dbDir => {
+        try {
+          logger.log(`Deleting orphaned database '${dbDir}'.`);
+          await fs.rmdir(dbDir, { recursive: true } as any);  // typings doesn't recognize the options argument
+        } catch (e) {
+          failures.push(`${path.basename(dbDir)}`);
+        }
+      })
+    );
+
+    if (failures.length) {
+      const dirname = path.dirname(failures[0]);
+      showAndLogErrorMessage(
+        `Failed to delete unused databases:\n  ${
+        failures.join('\n  ')
+        }\n. To delete unused databases, please remove them manually from the storage folder ${dirname}.`
+      );
+    }
+  };
+
+
   handleChooseDatabaseArchive = async (
     progress: ProgressCallback,
     token: CancellationToken
@@ -653,7 +711,7 @@ export class DatabaseUI extends DisposableObject {
       dbPath = path.dirname(dbPath);
     }
 
-    if (isLikelyDbFolder(dbPath)) {
+    if (isLikelyDbLanguageFolder(dbPath)) {
       dbPath = path.dirname(dbPath);
     }
     return Uri.file(dbPath);
@@ -668,9 +726,3 @@ export class DatabaseUI extends DisposableObject {
     }
   }
 }
-
-// TODO: Get the list of supported languages from a list that will be auto-updated.
-const dbRegeEx = /^db-(javascript|go|cpp|java|python|csharp)$/;
-function isLikelyDbFolder(dbPath: string) {
-  return path.basename(dbPath).match(dbRegeEx);
-}

extensions/ql-vscode/src/databases-ui.ts

@@ -397,10 +397,7 @@ export class DatabaseItemImpl implements DatabaseItem {
    * Holds if the database item refers to an exported snapshot
    */
   public async hasMetadataFile(): Promise<boolean> {
-    return (await Promise.all([
-      fs.pathExists(path.join(this.databaseUri.fsPath, '.dbinfo')),
-      fs.pathExists(path.join(this.databaseUri.fsPath, 'codeql-database.yml'))
-    ])).some(x => x);
+    return await isLikelyDatabaseRoot(this.databaseUri.fsPath);
   }
 
   /**
@@ -730,3 +727,23 @@ export function getUpgradesDirectories(scripts: string[]): vscode.Uri[] {
   const uniqueParentDirs = new Set(parentDirs);
   return Array.from(uniqueParentDirs).map(filePath => vscode.Uri.file(filePath));
 }
+
+
+// TODO: Get the list of supported languages from a list that will be auto-updated.
+
+export async function isLikelyDatabaseRoot(fsPath: string) {
+  const [a, b, c] = (await Promise.all([
+    // databases can have either .dbinfo or codeql-database.yml.
+    fs.pathExists(path.join(fsPath, '.dbinfo')),
+    fs.pathExists(path.join(fsPath, 'codeql-database.yml')),
+
+    // they *must* have a db-language folder
+    (await fs.readdir(fsPath)).some(isLikelyDbLanguageFolder)
+  ]));
+
+  return (a || b) && c;
+}
+
+export function isLikelyDbLanguageFolder(dbPath: string) {
+  return !!path.basename(dbPath).startsWith('db-');
+}

extensions/ql-vscode/src/databases.ts

@@ -349,6 +349,7 @@ async function activateWithInstalledDistribution(
     getContextStoragePath(ctx),
     ctx.extensionPath
   );
+  databaseUI.init();
   ctx.subscriptions.push(databaseUI);
 
   logger.log('Initializing query history manager.');
@@ -643,6 +644,8 @@ async function activateWithInstalledDistribution(
     title: 'Calculate AST'
   }));
 
+  commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
+
   logger.log('Successfully finished extension initialization.');
 }
 

extensions/ql-vscode/src/extension.ts

@@ -21,3 +21,11 @@ class ExhaustivityCheckingError extends Error {
 export function assertNever(value: never): never {
   throw new ExhaustivityCheckingError(value);
 }
+
+/**
+ * Use to perform array filters where the predicate is asynchronous.
+ */
+export const asyncFilter = async function <T>(arr: T[], predicate: (arg0: T) => Promise<boolean>) {
+  const results = await Promise.all(arr.map(predicate));
+  return arr.filter((_, index) => results[index]);
+};