Feed code search results into variant analysis repo lists

extensions/ql-vscode/package.json

@@ -516,6 +516,10 @@
         "title": "Add new list",
         "icon": "$(new-folder)"
       },
+      {
+        "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+        "title": "Import Repos from GitHub Code Search"
+      },
       {
         "command": "codeQLVariantAnalysisRepositories.setSelectedItem",
         "title": "Select"
@@ -961,6 +965,11 @@
           "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canBeOpenedOnGitHub/",
           "group": "2_qlContextMenu@1"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+          "when": "view == codeQLVariantAnalysisRepositories && viewItem =~ /canImportCodeSearch/",
+          "group": "2_qlContextMenu@1"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "group": "inline",
@@ -1297,6 +1306,10 @@
           "command": "codeQLVariantAnalysisRepositories.removeItemContextMenu",
           "when": "false"
         },
+        {
+          "command": "codeQLVariantAnalysisRepositories.importCodeSearch",
+          "when": "false"
+        },
         {
           "command": "codeQLDatabases.setCurrentDatabase",
           "when": "false"

extensions/ql-vscode/src/common/commands.ts

@@ -275,6 +275,7 @@ export type DatabasePanelCommands = {
   "codeQLVariantAnalysisRepositories.openOnGitHubContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.renameItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
   "codeQLVariantAnalysisRepositories.removeItemContextMenu": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
+  "codeQLVariantAnalysisRepositories.importCodeSearch": TreeViewContextSingleSelectionCommandFunction<DbTreeViewItem>;
 };
 
 export type AstCfgCommands = {

extensions/ql-vscode/src/databases/config/db-config-store.ts

@@ -145,10 +145,38 @@ export class DbConfigStore extends DisposableObject {
     await this.writeConfig(config);
   }
 
+  public async addRemoteReposToList(
+    repoNwoList: string[],
+    parentList: string,
+  ): Promise<string[]> {
+    if (!this.config) {
+      throw Error("Cannot add variant analysis repos if config is not loaded");
+    }
+
+    const config = cloneDbConfig(this.config);
+    const parent = config.databases.variantAnalysis.repositoryLists.find(
+      (list) => list.name === parentList,
+    );
+    if (!parent) {
+      throw Error(`Cannot find parent list '${parentList}'`);
+    }
+
+    // Remove duplicates from the list of repositories.
+    const newRepositoriesList = [
+      ...new Set([...new Set(parent.repositories), ...new Set(repoNwoList)]),
+    ];
+
+    parent.repositories = newRepositoriesList.slice(0, 1000);
+    const truncatedRepositories = newRepositoriesList.slice(1000);
+
+    await this.writeConfig(config);
+    return truncatedRepositories;
+  }
+
   public async addRemoteRepo(
     repoNwo: string,
     parentList?: string,
-  ): Promise<void> {
+  ): Promise<string[]> {
     if (!this.config) {
       throw Error("Cannot add variant analysis repo if config is not loaded");
     }
@@ -163,6 +191,7 @@ export class DbConfigStore extends DisposableObject {
       );
     }
 
+    const truncatedRepositories = [];
     const config = cloneDbConfig(this.config);
     if (parentList) {
       const parent = config.databases.variantAnalysis.repositoryLists.find(
@@ -171,12 +200,15 @@ export class DbConfigStore extends DisposableObject {
       if (!parent) {
         throw Error(`Cannot find parent list '${parentList}'`);
       } else {
-        parent.repositories.push(repoNwo);
+        const newRepositories = [...parent.repositories, repoNwo];
+        parent.repositories = newRepositories.slice(0, 1000);
+        truncatedRepositories.push(...newRepositories.slice(1000));
       }
     } else {
       config.databases.variantAnalysis.repositories.push(repoNwo);
     }
     await this.writeConfig(config);
+    return truncatedRepositories;
   }
 
   public async addRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/db-manager.ts

@@ -96,8 +96,15 @@ export class DbManager {
   public async addNewRemoteRepo(
     nwo: string,
     parentList?: string,
-  ): Promise<void> {
-    await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteRepo(nwo, parentList);
+  }
+
+  public async addNewRemoteReposToList(
+    nwoList: string[],
+    parentList: string,
+  ): Promise<string[]> {
+    return await this.dbConfigStore.addRemoteReposToList(nwoList, parentList);
   }
 
   public async addNewRemoteOwner(owner: string): Promise<void> {

extensions/ql-vscode/src/databases/ui/db-panel.ts

@@ -32,6 +32,8 @@ import { getControllerRepo } from "../../variant-analysis/run-remote-query";
 import { getErrorMessage } from "../../pure/helpers-pure";
 import { DatabasePanelCommands } from "../../common/commands";
 import { App } from "../../common/app";
+import { getCodeSearchRepositories } from "../../variant-analysis/gh-api/gh-api-client";
+import { QueryLanguage } from "../../common/query-language";
 
 export interface RemoteDatabaseQuickPickItem extends QuickPickItem {
   remoteDatabaseKind: string;
@@ -41,6 +43,10 @@ export interface AddListQuickPickItem extends QuickPickItem {
   databaseKind: DbListKind;
 }
 
+export interface CodeSearchQuickPickItem extends QuickPickItem {
+  language: string;
+}
+
 export class DbPanel extends DisposableObject {
   private readonly dataProvider: DbTreeDataProvider;
   private readonly treeView: TreeView<DbTreeViewItem>;
@@ -93,6 +99,8 @@ export class DbPanel extends DisposableObject {
         this.renameItem.bind(this),
       "codeQLVariantAnalysisRepositories.removeItemContextMenu":
         this.removeItem.bind(this),
+      "codeQLVariantAnalysisRepositories.importCodeSearch":
+        this.importCodeSearch.bind(this),
     };
   }
 
@@ -171,7 +179,14 @@ export class DbPanel extends DisposableObject {
       return;
     }
 
-    await this.dbManager.addNewRemoteRepo(nwo, parentList);
+    const truncatedRepositories = await this.dbManager.addNewRemoteRepo(
+      nwo,
+      parentList,
+    );
+
+    if (parentList) {
+      this.truncatedReposNote(truncatedRepositories, parentList);
+    }
   }
 
   private async addNewRemoteOwner(): Promise<void> {
@@ -323,6 +338,71 @@ export class DbPanel extends DisposableObject {
     await this.dbManager.removeDbItem(treeViewItem.dbItem);
   }
 
+  private async importCodeSearch(treeViewItem: DbTreeViewItem): Promise<void> {
+    if (treeViewItem.dbItem?.kind !== DbItemKind.RemoteUserDefinedList) {
+      throw new Error("Please select a valid list to add code search results.");
+    }
+
+    const languageQuickPickItems: CodeSearchQuickPickItem[] = Object.values(
+      QueryLanguage,
+    ).map((language) => ({
+      label: language.toString(),
+      alwaysShow: true,
+      language: language.toString(),
+    }));
+
+    const codeSearchLanguage =
+      await window.showQuickPick<CodeSearchQuickPickItem>(
+        languageQuickPickItems,
+        {
+          title: "Select the language you want to query",
+          placeHolder: "Select an option",
+          ignoreFocusOut: true,
+        },
+      );
+    if (!codeSearchLanguage) {
+      // We don't need to display a warning pop-up in this case, since the user just escaped out of the operation.
+      // We set 'true' to make this a silent exception.
+      throw new UserCancellationException("No language selected", true);
+    }
+
+    const codeSearchQuery = await window.showInputBox({
+      title: "Code search query",
+      prompt: "Insert code search query",
+      placeHolder: "org:github",
+    });
+    if (codeSearchQuery === undefined || codeSearchQuery === "") {
+      return;
+    }
+
+    const repositories = await getCodeSearchRepositories(
+      this.app.credentials,
+      `${codeSearchQuery} language:${codeSearchLanguage.language}`,
+    );
+
+    const truncatedRepositories = await this.dbManager.addNewRemoteReposToList(
+      repositories,
+      treeViewItem.dbItem.listName,
+    );
+    this.truncatedReposNote(
+      truncatedRepositories,
+      treeViewItem.dbItem.listName,
+    );
+  }
+
+  private truncatedReposNote(
+    truncatedRepositories: string[],
+    listName: string,
+  ) {
+    if (truncatedRepositories.length > 0) {
+      void showAndLogErrorMessage(
+        `Some repositories were not added to '${listName}' because a list can only have 1000 entries. Excluded repositories: ${truncatedRepositories.join(
+          ", ",
+        )}`,
+      );
+    }
+  }
+
   private async onDidCollapseElement(
     event: TreeViewExpansionEvent<DbTreeViewItem>,
   ): Promise<void> {

extensions/ql-vscode/src/databases/ui/db-tree-view-item-action.ts

@@ -4,7 +4,8 @@ export type DbTreeViewItemAction =
   | "canBeSelected"
   | "canBeRemoved"
   | "canBeRenamed"
-  | "canBeOpenedOnGitHub";
+  | "canBeOpenedOnGitHub"
+  | "canImportCodeSearch";
 
 export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
   const actions: DbTreeViewItemAction[] = [];
@@ -21,7 +22,9 @@ export function getDbItemActions(dbItem: DbItem): DbTreeViewItemAction[] {
   if (canBeOpenedOnGitHub(dbItem)) {
     actions.push("canBeOpenedOnGitHub");
   }
-
+  if (canImportCodeSearch(dbItem)) {
+    actions.push("canImportCodeSearch");
+  }
   return actions;
 }
 
@@ -60,6 +63,10 @@ function canBeOpenedOnGitHub(dbItem: DbItem): boolean {
   return dbItemKindsThatCanBeOpenedOnGitHub.includes(dbItem.kind);
 }
 
+function canImportCodeSearch(dbItem: DbItem): boolean {
+  return DbItemKind.RemoteUserDefinedList === dbItem.kind;
+}
+
 export function getGitHubUrl(dbItem: DbItem): string | undefined {
   switch (dbItem.kind) {
     case DbItemKind.RemoteOwner:

extensions/ql-vscode/src/variant-analysis/gh-api/gh-api-client.ts

@@ -8,6 +8,24 @@ import {
 } from "./variant-analysis";
 import { Repository } from "./repository";
 
+export async function getCodeSearchRepositories(
+  credentials: Credentials,
+  query: string,
+): Promise<string[]> {
+  const octokit = await credentials.getOctokit();
+
+  const nwos = await octokit.paginate(
+    octokit.rest.search.repos,
+    {
+      q: query,
+      per_page: 100,
+    },
+    (response) => response.data.map((item) => item.full_name),
+  );
+
+  return [...new Set(nwos)];
+}
+
 export async function submitVariantAnalysis(
   credentials: Credentials,
   submissionDetails: VariantAnalysisSubmission,