Integrated CLI database unbundle command for archive download

Co-authored by: Marc Jaramillo mnj.webdeveloper@gmail.comm
Co-authored by: Musab Guma'a mgsium@github.com

Author: marcnjaramillo

extensions/ql-vscode/src/cli.ts

@@ -600,6 +600,15 @@ export class CodeQLCliServer implements Disposable {
     return await this.runJsonCodeQlCliCommand<BQRSInfo>(['bqrs', 'info'], subcommandArgs, 'Reading bqrs header');
   }
 
+  async databaseUnbundle(archivePath: string, target: string, name?: string): Promise<string> {
+    const subcommandArgs = [];
+    if (target) subcommandArgs.push('--target', target);
+    if (name) subcommandArgs.push('--name', name);
+    subcommandArgs.push(archivePath);
+ 
+    return await this.runCodeQlCliCommand(['database', 'unbundle'], subcommandArgs, `Extracting ${archivePath} to directory ${target}`);
+  }
+
   /**
   * Gets the results from a bqrs.
   * @param bqrsPath The path to the bqrs.

extensions/ql-vscode/src/databaseFetcher.ts

@@ -1,12 +1,12 @@
 import fetch, { Response } from 'node-fetch';
-import * as unzipper from 'unzipper';
 import { zip } from 'zip-a-folder';
 import {
   Uri,
   CancellationToken,
   commands,
   window,
 } from 'vscode';
+import {CodeQLCliServer} from './cli';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 
@@ -28,6 +28,7 @@ import { tmpDir } from './run-queries';
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportInternetDatabase(
+  cli: CodeQLCliServer,
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
@@ -43,6 +44,7 @@ export async function promptImportInternetDatabase(
   validateHttpsUrl(databaseUrl);
 
   const item = await databaseArchiveFetcher(
+    cli,
     databaseUrl,
     databaseManager,
     storagePath,
@@ -67,6 +69,7 @@ export async function promptImportInternetDatabase(
  * @param storagePath where to store the unzipped database.
  */
 export async function promptImportLgtmDatabase(
+  cli: CodeQLCliServer,
   databaseManager: DatabaseManager,
   storagePath: string,
   progress: ProgressCallback,
@@ -89,6 +92,7 @@ export async function promptImportLgtmDatabase(
     const databaseUrl = await convertToDatabaseUrl(lgtmUrl, progress);
     if (databaseUrl) {
       const item = await databaseArchiveFetcher(
+        cli,
         databaseUrl,
         databaseManager,
         storagePath,
@@ -115,6 +119,7 @@ export async function promptImportLgtmDatabase(
  * @param storagePath where to store the unzipped database.
  */
 export async function importArchiveDatabase(
+  cli: CodeQLCliServer,
   databaseUrl: string,
   databaseManager: DatabaseManager,
   storagePath: string,
@@ -123,6 +128,7 @@ export async function importArchiveDatabase(
 ): Promise<DatabaseItem | undefined> {
   try {
     const item = await databaseArchiveFetcher(
+      cli,
       databaseUrl,
       databaseManager,
       storagePath,
@@ -155,6 +161,7 @@ export async function importArchiveDatabase(
  * @param token cancellation token
  */
 async function databaseArchiveFetcher(
+  cli: CodeQLCliServer,
   databaseUrl: string,
   databaseManager: DatabaseManager,
   storagePath: string,
@@ -173,9 +180,9 @@ async function databaseArchiveFetcher(
   const unzipPath = await getStorageFolder(storagePath, databaseUrl);
 
   if (isFile(databaseUrl)) {
-    await readAndUnzip(databaseUrl, unzipPath, progress);
+    await readAndUnzip(cli, databaseUrl, unzipPath, progress);
   } else {
-    await fetchAndUnzip(databaseUrl, unzipPath, progress);
+    await fetchAndUnzip(cli, databaseUrl, unzipPath, progress);
   }
 
   progress({
@@ -247,6 +254,7 @@ function validateHttpsUrl(databaseUrl: string) {
 }
 
 async function readAndUnzip(
+  cli: CodeQLCliServer,
   zipUrl: string,
   unzipPath: string,
   progress?: ProgressCallback
@@ -262,11 +270,11 @@ async function readAndUnzip(
   // Must get the zip central directory since streaming the
   // zip contents may not have correct local file headers.
   // Instead, we can only rely on the central directory.
-  const directory = await unzipper.Open.file(zipFile);
-  await directory.extract({ path: unzipPath });
+  await cli.databaseUnbundle(zipFile, unzipPath);
 }
 
 async function fetchAndUnzip(
+  cli: CodeQLCliServer,
   databaseUrl: string,
   unzipPath: string,
   progress?: ProgressCallback
@@ -298,7 +306,7 @@ async function fetchAndUnzip(
       .on('error', reject)
   );
 
-  await readAndUnzip(Uri.file(archivePath).toString(true), unzipPath, progress);
+  await readAndUnzip(cli, Uri.file(archivePath).toString(true), unzipPath, progress);
 
   // remove archivePath eagerly since these archives can be large.
   await fs.remove(archivePath);

extensions/ql-vscode/src/databases-ui.ts

@@ -454,7 +454,11 @@ export class DatabaseUI extends DisposableObject {
   ): Promise<
     DatabaseItem | undefined
   > => {
+    if (!this.queryServer) {
+      throw new Error('Query server not started');
+    }
     return await promptImportInternetDatabase(
+      this.queryServer.cliServer,
       this.databaseManager,
       this.storagePath,
       progress,
@@ -466,7 +470,11 @@ export class DatabaseUI extends DisposableObject {
     progress: ProgressCallback,
     token: CancellationToken
   ): Promise<DatabaseItem | undefined> => {
+    if (!this.queryServer) {
+      throw new Error('Query server not started');
+    }
     return await promptImportLgtmDatabase(
+      this.queryServer.cliServer,
       this.databaseManager,
       this.storagePath,
       progress,
@@ -572,10 +580,15 @@ export class DatabaseUI extends DisposableObject {
     token: CancellationToken,
     uri: Uri,
   ): Promise<void> => {
+    if (!this.queryServer) {
+      throw new Error('Query server not started');
+    }
+
     try {
       // Assume user has selected an archive if the file has a .zip extension
       if (uri.path.endsWith('.zip')) {
         await importArchiveDatabase(
+          this.queryServer.cliServer,
           uri.toString(true),
           this.databaseManager,
           this.storagePath,
@@ -695,8 +708,11 @@ export class DatabaseUI extends DisposableObject {
     progress: ProgressCallback,
     token: CancellationToken,
   ): Promise<DatabaseItem | undefined> {
-    const uri = await chooseDatabaseDir(byFolder);
+    if (!this.queryServer) {
+      throw new Error('Query server not started');
+    }
 
+    const uri = await chooseDatabaseDir(byFolder);
     if (!uri) {
       return undefined;
     }
@@ -709,6 +725,7 @@ export class DatabaseUI extends DisposableObject {
       // we are selecting a database archive. Must unzip into a workspace-controlled area
       // before importing.
       return await importArchiveDatabase(
+        this.queryServer.cliServer,
         uri.toString(true),
         this.databaseManager,
         this.storagePath,

extensions/ql-vscode/src/vscode-tests/cli-integration/databases.test.ts

@@ -5,6 +5,7 @@ import { expect } from 'chai';
 import { extensions, CancellationToken, Uri, window } from 'vscode';
 
 import { CodeQLExtensionInterface } from '../../extension';
+import { CodeQLCliServer } from '../../cli';
 import { DatabaseManager } from '../../databases';
 import { promptImportLgtmDatabase, importArchiveDatabase, promptImportInternetDatabase } from '../../databaseFetcher';
 import { ProgressCallback } from '../../commandRunner';
@@ -17,7 +18,8 @@ describe('Databases', function() {
   this.timeout(60000);
 
   const LGTM_URL = 'https://lgtm.com/projects/g/aeisenberg/angular-bind-notifier/';
-
+  
+  let cli: CodeQLCliServer;
   let databaseManager: DatabaseManager;
   let sandbox: sinon.SinonSandbox;
   let inputBoxStub: sinon.SinonStub;
@@ -53,7 +55,7 @@ describe('Databases', function() {
   it('should add a database from a folder', async () => {
     const progressCallback = sandbox.spy() as ProgressCallback;
     const uri = Uri.file(dbLoc);
-    let dbItem = await importArchiveDatabase(uri.toString(true), databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    let dbItem = await importArchiveDatabase(cli, uri.toString(true), databaseManager, storagePath, progressCallback, {} as CancellationToken);
     expect(dbItem).to.be.eq(databaseManager.currentDatabaseItem);
     expect(dbItem).to.be.eq(databaseManager.databaseItems[0]);
     expect(dbItem).not.to.be.undefined;
@@ -64,7 +66,7 @@ describe('Databases', function() {
 
   it('should add a database from lgtm with only one language', async () => {
     inputBoxStub.resolves(LGTM_URL);
-    let dbItem = await promptImportLgtmDatabase(databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    let dbItem = await promptImportLgtmDatabase(cli, databaseManager, storagePath, progressCallback, {} as CancellationToken);
     expect(dbItem).not.to.be.undefined;
     dbItem = dbItem!;
     expect(dbItem.name).to.eq('aeisenberg_angular-bind-notifier_106179a');
@@ -74,7 +76,7 @@ describe('Databases', function() {
   it('should add a database from a url', async () => {
     inputBoxStub.resolves(DB_URL);
 
-    let dbItem = await promptImportInternetDatabase(databaseManager, storagePath, progressCallback, {} as CancellationToken);
+    let dbItem = await promptImportInternetDatabase(cli, databaseManager, storagePath, progressCallback, {} as CancellationToken);
     expect(dbItem).not.to.be.undefined;
     dbItem = dbItem!;
     expect(dbItem.name).to.eq('db');

extensions/ql-vscode/src/vscode-tests/cli-integration/queries.test.ts

@@ -64,6 +64,7 @@ describe('Queries', function() {
       // Add a database
       const uri = Uri.file(dbLoc);
       const maybeDbItem = await importArchiveDatabase(
+        cli,
         uri.toString(true),
         databaseManager,
         storagePath,