Use codeql resolve database to get language

This commit moves to using codeql resolve database instead of inspecting
the `codeql-database.yml` file.

When the extension starts and if the cli supports it, the extension will
attempt to get the name for any databases that don't yet have a name.
Once a name is searched for once by the cli, it will be cached so we
don't need to rediscover the name again.

Author: aeisenberg

extensions/ql-vscode/src/cli.ts

@@ -50,7 +50,7 @@ export interface DbInfo {
   sourceArchiveRoot: string;
   datasetFolder: string;
   logsFolder: string;
-  primaryLanguage: string;
+  languages: string[];
 }
 
 /**
@@ -124,6 +124,11 @@ export class CodeQLCliServer implements Disposable {
    */
   private static CLI_VERSION_WITH_DECOMPILE_KIND_DIL = new SemVer('2.3.0');
 
+  /**
+   * CLI version where --kind=DIL was introduced
+   */
+  private static CLI_VERSION_WITH_LANGUAGE = new SemVer('2.4.1');
+
   /** The process for the cli server, or undefined if one doesn't exist yet */
   process?: child_process.ChildProcessWithoutNullStreams;
   /** Queue of future commands*/
@@ -690,7 +695,7 @@ export class CodeQLCliServer implements Disposable {
   }
 
   async generateDil(qloFile: string, outFile: string): Promise<void> {
-    const extraArgs = (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_DECOMPILE_KIND_DIL) >= 0
+    const extraArgs = await this.supportsDecompileDil()
       ? ['--kind', 'dil', '-o', outFile, qloFile]
       : ['-o', outFile, qloFile];
     await this.runCodeQlCliCommand(
@@ -707,6 +712,14 @@ export class CodeQLCliServer implements Disposable {
     return this._version;
   }
 
+  private async supportsDecompileDil() {
+    return (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_DECOMPILE_KIND_DIL) >= 0;
+  }
+
+  public async supportsLangaugeName() {
+    return (await this.getVersion()).compare(CodeQLCliServer.CLI_VERSION_WITH_LANGUAGE) >= 0;
+  }
+
   private async refreshVersion() {
     const distribution = await this.distributionProvider.getDistribution();
     switch (distribution.kind) {

extensions/ql-vscode/src/databases.ts

@@ -8,7 +8,6 @@ import {
   showAndLogErrorMessage,
   showAndLogWarningMessage,
   showAndLogInformationMessage,
-  getPrimaryLanguage,
   isLikelyDatabaseRoot,
   ProgressCallback,
   withProgress
@@ -51,7 +50,7 @@ export interface DatabaseOptions {
 export interface FullDatabaseOptions extends DatabaseOptions {
   ignoreSourceArchive: boolean;
   dateAdded: number | undefined;
-  language: string;
+  language: string | undefined;
 }
 
 interface PersistedDatabaseItem {
@@ -508,7 +507,8 @@ export class DatabaseManager extends DisposableObject {
   constructor(
     private readonly ctx: ExtensionContext,
     private readonly qs: QueryServerClient,
-    public readonly logger: Logger
+    private readonly cli: cli.CodeQLCliServer,
+    public logger: Logger
   ) {
     super();
 
@@ -528,7 +528,7 @@ export class DatabaseManager extends DisposableObject {
       // displayName is only set if a user explicitly renames a database
       displayName: undefined,
       dateAdded: Date.now(),
-      language: await getPrimaryLanguage(uri.fsPath)
+      language: await this.getPrimaryLanguage(uri.fsPath)
     };
     const databaseItem = new DatabaseItemImpl(uri, contents, fullOptions, (event) => {
       this._onDidChangeDatabaseItem.fire(event);
@@ -588,7 +588,7 @@ export class DatabaseManager extends DisposableObject {
     let displayName: string | undefined = undefined;
     let ignoreSourceArchive = false;
     let dateAdded = undefined;
-    let language = '';
+    let language = undefined;
     if (state.options) {
       if (typeof state.options.displayName === 'string') {
         displayName = state.options.displayName;
@@ -599,9 +599,13 @@ export class DatabaseManager extends DisposableObject {
       if (typeof state.options.dateAdded === 'number') {
         dateAdded = state.options.dateAdded;
       }
-      if (state.options.language) {
-        language = state.options.language;
-      }
+      language = state.options.language;
+    }
+
+    const dbBaseUri = vscode.Uri.parse(state.uri, true);
+    if (language === undefined) {
+      // we haven't been successful yet at getting the language. try again
+      language = await this.getPrimaryLanguage(dbBaseUri.fsPath);
     }
 
     const fullOptions: FullDatabaseOptions = {
@@ -610,7 +614,7 @@ export class DatabaseManager extends DisposableObject {
       dateAdded,
       language
     };
-    const item = new DatabaseItemImpl(vscode.Uri.parse(state.uri, true), undefined, fullOptions,
+    const item = new DatabaseItemImpl(dbBaseUri, undefined, fullOptions,
       (event) => {
         this._onDidChangeDatabaseItem.fire(event);
       });
@@ -825,6 +829,15 @@ export class DatabaseManager extends DisposableObject {
     }
     return false;
   }
+
+  private async getPrimaryLanguage(dbPath: string) {
+    if (!(await this.cli.supportsLangaugeName())) {
+      // return undefined so that we continually recalculate until the cli version is bumped
+      return undefined;
+    }
+    const dbInfo = await this.cli.resolveDatabase(dbPath);
+    return dbInfo.languages?.[0] || '';
+  }
 }
 
 /**

extensions/ql-vscode/src/extension.ts

@@ -339,7 +339,7 @@ async function activateWithInstalledDistribution(
   await qs.startQueryServer();
 
   logger.log('Initializing database manager.');
-  const dbm = new DatabaseManager(ctx, qs, logger);
+  const dbm = new DatabaseManager(ctx, qs, cliServer, logger);
   ctx.subscriptions.push(dbm);
   logger.log('Initializing database panel.');
   const databaseUI = new DatabaseUI(

extensions/ql-vscode/src/helpers.ts

@@ -464,6 +464,16 @@ export class CachedOperation<U> {
  * The following functions al heuristically determine metadata about databases.
  */
 
+/**
+ * Note that this heuristic is only being used for backwards compatibility with
+ * CLI versions before the langauge name was introduced to dbInfo. Implementations
+ * that do not require backwards compatibility should call
+ * `cli.CodeQLCliServer.resolveDatabase` and use the first entry in the
+ * `languages` property.
+ *
+ * @see cli.CodeQLCliServer.supportsLangaugeName
+ * @see cli.CodeQLCliServer.resolveDatabase
+ */
 const dbSchemeToLanguage = {
   'semmlecode.javascript.dbscheme': 'javascript',
   'semmlecode.cpp.dbscheme': 'cpp',
@@ -496,6 +506,12 @@ export function getInitialQueryContents(language: string, dbscheme: string) {
     : 'select ""';
 }
 
+/**
+ * Heuristically determines if the directory passed in corresponds
+ * to a database root.
+ *
+ * @param maybeRoot
+ */
 export async function isLikelyDatabaseRoot(maybeRoot: string) {
   const [a, b, c] = (await Promise.all([
     // databases can have either .dbinfo or codeql-database.yml.
@@ -512,16 +528,3 @@ export async function isLikelyDatabaseRoot(maybeRoot: string) {
 export function isLikelyDbLanguageFolder(dbPath: string) {
   return !!path.basename(dbPath).startsWith('db-');
 }
-
-export async function getPrimaryLanguage(root: string) {
-  try {
-    const metadataFile = path.join(root, 'codeql-database.yml');
-    if (await fs.pathExists(metadataFile)) {
-      const metadata = yaml.safeLoad(await fs.readFile(metadataFile, 'utf8')) as { primaryLanguage: string | undefined };
-      return metadata.primaryLanguage || '';
-    }
-  } catch (e) {
-    // could not determine language
-  }
-  return '';
-}

extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts

@@ -15,10 +15,11 @@ import {
   FullDatabaseOptions
 } from '../../databases';
 import { Logger } from '../../logging';
-import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
 import { QueryServerClient } from '../../queryserver-client';
 import { registerDatabases } from '../../pure/messages';
-import { isLikelyDbLanguageFolder, ProgressCallback } from '../../helpers';
+import { ProgressCallback } from '../../helpers';
+import { CodeQLCliServer } from '../../cli';
+import { encodeArchiveBasePath, encodeSourceArchiveUri } from '../../archive-filesystem-provider';
 
 describe('databases', () => {
 
@@ -34,6 +35,8 @@ describe('databases', () => {
   let dbChangedHandler: sinon.SinonSpy;
   let sendRequestSpy: sinon.SinonSpy;
   let supportsDatabaseRegistrationSpy: sinon.SinonStub;
+  let supportsLanguageNameSpy: sinon.SinonStub;
+  let resolveDatabaseSpy: sinon.SinonStub;
 
   let sandbox: sinon.SinonSandbox;
   let dir: tmp.DirResult;
@@ -51,6 +54,8 @@ describe('databases', () => {
     dbChangedHandler = sandbox.spy();
     supportsDatabaseRegistrationSpy = sandbox.stub();
     supportsDatabaseRegistrationSpy.resolves(true);
+    supportsLanguageNameSpy = sandbox.stub();
+    resolveDatabaseSpy = sandbox.stub();
     databaseManager = new DatabaseManager(
       {
         workspaceState: {
@@ -65,6 +70,10 @@ describe('databases', () => {
         sendRequest: sendRequestSpy,
         supportsDatabaseRegistration: supportsDatabaseRegistrationSpy
       } as unknown as QueryServerClient,
+      {
+        supportsLangaugeName: supportsLanguageNameSpy,
+        resolveDatabase: resolveDatabaseSpy
+      } as unknown as CodeQLCliServer,
       {} as Logger,
     );
 
@@ -377,9 +386,29 @@ describe('databases', () => {
     });
   });
 
-  it('should find likely db language folders', () => {
-    expect(isLikelyDbLanguageFolder('db-javascript')).to.be.true;
-    expect(isLikelyDbLanguageFolder('dbnot-a-db')).to.be.false;
+  it('should not support the primary language', async () => {
+    supportsLanguageNameSpy.resolves(false);
+
+    const result = (await (databaseManager as any).getPrimaryLanguage('hucairz'));
+    expect(result).to.be.undefined;
+  });
+
+  it('should get the primary language', async () => {
+    supportsLanguageNameSpy.resolves(true);
+    resolveDatabaseSpy.resolves({
+      languages: ['python']
+    });
+    const result = (await (databaseManager as any).getPrimaryLanguage('hucairz'));
+    expect(result).to.eq('python');
+  });
+
+  it('should handle missing the primary language', async () => {
+    supportsLanguageNameSpy.resolves(true);
+    resolveDatabaseSpy.resolves({
+      languages: []
+    });
+    const result = (await (databaseManager as any).getPrimaryLanguage('hucairz'));
+    expect(result).to.eq('');
   });
 
   function createMockDB(

extensions/ql-vscode/src/vscode-tests/no-workspace/helpers.test.ts

@@ -6,7 +6,7 @@ import * as tmp from 'tmp';
 import * as path from 'path';
 import * as fs from 'fs-extra';
 
-import { getInitialQueryContents, getPrimaryLanguage, InvocationRateLimiter } from '../../helpers';
+import { getInitialQueryContents, InvocationRateLimiter, isLikelyDbLanguageFolder } from '../../helpers';
 
 describe('Invocation rate limiter', () => {
   // 1 January 2020
@@ -105,14 +105,6 @@ describe('codeql-database.yml tests', () => {
     dir.removeCallback();
   });
 
-  it('should get the language of a database', async () => {
-    expect(await getPrimaryLanguage(dir.name)).to.eq('cpp');
-  });
-
-  it('should get the language of a database when langauge is not known', async () => {
-    expect(await getPrimaryLanguage('xxx')).to.eq('');
-  });
-
   it('should get initial query contents when language is known', () => {
     expect(getInitialQueryContents('cpp', 'hucairz')).to.eq('import cpp\n\nselect ""');
   });
@@ -126,6 +118,11 @@ describe('codeql-database.yml tests', () => {
   });
 });
 
+it('should find likely db language folders', () => {
+  expect(isLikelyDbLanguageFolder('db-javascript')).to.be.true;
+  expect(isLikelyDbLanguageFolder('dbnot-a-db')).to.be.false;
+});
+
 class MockExtensionContext implements ExtensionContext {
   subscriptions: { dispose(): unknown }[] = [];
   workspaceState: Memento = new MockMemento();