beam_arxiv_scrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "66abf160",
   "metadata": {
    "id": "66abf160"
   },
   "outputs": [],
   "source": [
    "import apache_beam as beam\n",
    "import arxiv \n",
    "\n",
    "from apache_beam.dataframe.convert import to_dataframe\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5443193c",
   "metadata": {
    "id": "5443193c"
   },
   "outputs": [],
   "source": [
    "query_keywords = [\n",
    "    \"\\\"image segmentation\\\"\",\n",
    "    \"\\\"self-supervised learning\\\"\",\n",
    "    \"\\\"representation learning\\\"\",\n",
    "    \"\\\"image generation\\\"\",\n",
    "    \"\\\"object detection\\\"\",\n",
    "    \"\\\"transfer learning\\\"\",\n",
    "    \"\\\"transformers\\\"\",\n",
    "    \"\\\"adversarial training\",\n",
    "    \"\\\"generative adversarial networks\\\"\",\n",
    "    \"\\\"model compressions\\\"\",\n",
    "    \"\\\"image segmentation\\\"\",\n",
    "    \"\\\"few-shot learning\\\"\",\n",
    "    \"\\\"natural language\\\"\",\n",
    "    \"\\\"graph\\\"\",\n",
    "    \"\\\"colorization\\\"\",\n",
    "    \"\\\"depth estimation\\\"\",\n",
    "    \"\\\"point cloud\\\"\",\n",
    "    \"\\\"structured data\\\"\",\n",
    "    \"\\\"optical flow\\\"\",\n",
    "    \"\\\"reinforcement learning\\\"\",\n",
    "    \"\\\"super resolution\\\"\",\n",
    "    \"\\\"attention\\\"\",\n",
    "    \"\\\"tabular\\\"\",\n",
    "    \"\\\"unsupervised learning\\\"\",\n",
    "    \"\\\"semi-supervised learning\\\"\",\n",
    "    \"\\\"explainable\\\"\",\n",
    "    \"\\\"radiance field\\\"\",\n",
    "    \"\\\"decision tree\\\"\",\n",
    "    \"\\\"time series\\\"\",\n",
    "    \"\\\"molecule\\\"\",\n",
    "    \"\\\"physics\\\"\",\n",
    "    \"\\\"graphics\\\"\",\n",
    "    \"\\\"ray tracing\\\"\",\n",
    "    \"\\\"optical flow\\\"\",\n",
    "    \"\\\"photogrametry\\\"\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "DI84CCdnY5Ek",
   "metadata": {
    "id": "DI84CCdnY5Ek"
   },
   "outputs": [],
   "source": [
    "import typing\n",
    "\n",
    "\n",
    "class ArxivEntries(typing.NamedTuple):\n",
    "    terms: typing.List[str]\n",
    "    titles: str\n",
    "    abstracts: str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "83ddf9d6",
   "metadata": {
    "id": "83ddf9d6"
   },
   "outputs": [],
   "source": [
    "client = arxiv.Client(num_retries=20, page_size=500)\n",
    "\n",
    "\n",
    "def query_with_keywords(query):\n",
    "    search = arxiv.Search(\n",
    "        query=query, max_results=20000, sort_by=arxiv.SortCriterion.LastUpdatedDate,\n",
    "    )\n",
    "\n",
    "    for res in client.results(search):\n",
    "        if res.primary_category in [\"cs.CV\", \"stat.ML\", \"cs.LG\"]:\n",
    "            yield beam.Row(\n",
    "                terms=res.categories, titles=res.title, abstracts=res.summary\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "SqeUCclZWwF1",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SqeUCclZWwF1",
    "outputId": "e086cff2-8660-4079-9128-66839909a009"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting setup.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile setup.py\n",
    "\n",
    "import setuptools\n",
    "\n",
    "\n",
    "NAME = \"gather_arxiv_data\"\n",
    "VERSION = \"0.1.0\"\n",
    "REQUIRED_PACKAGES = [\n",
    "    \"apache_beam==2.32.0\",\n",
    "    \"pandas==1.3.2\",\n",
    "    \"arxiv==1.4.2\",\n",
    "    \"google_cloud_storage==1.42.1\",\n",
    "]\n",
    "\n",
    "\n",
    "setuptools.setup(\n",
    "    name=NAME,\n",
    "    version=VERSION,\n",
    "    install_requires=REQUIRED_PACKAGES,\n",
    "    packages=setuptools.find_packages(),\n",
    "    include_package_data=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2196906b",
   "metadata": {
    "id": "2196906b"
   },
   "outputs": [],
   "source": [
    "gcs_bucket_name = \"arxiv-data-nlp\"\n",
    "gcp_project = \"####\" # Specify this.\n",
    "\n",
    "pipeline_args = {\n",
    "    \"job_name\": f'arxiv-data-{datetime.utcnow().strftime(\"%y%m%d-%H%M%S\")}',\n",
    "    \"num_workers\": \"4\",\n",
    "    \"runner\": \"DataflowRunner\",\n",
    "    \"setup_file\": \"./setup.py\",\n",
    "    \"project\": gcp_project,\n",
    "    \"region\": \"us-central1\",\n",
    "    \"gcs_location\": f\"gs://{gcs_bucket_name}\",\n",
    "    \"temp_location\": f\"gs://{gcs_bucket_name}/temp\",\n",
    "    \"staging_location\": f\"gs://{gcs_bucket_name}/staging\",\n",
    "    \"save_main_session\": \"True\",\n",
    "}\n",
    "\n",
    "# Convert the dictionary to a list of (argument, value) tuples and then flatten the list.\n",
    "pipeline_args = [(f\"--{k}\", v) for k, v in pipeline_args.items()]\n",
    "pipeline_args = [x for y in pipeline_args for x in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d58affea",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 766
    },
    "id": "d58affea",
    "outputId": "23852f9e-bf43-4b2a-cd3a-6180761a904f"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:apache_beam.runners.interactive.interactive_environment:Dependencies required for Interactive Beam PCollection visualization are not available, please use: `pip install apache-beam[interactive]` to install necessary dependencies to enable all data visualization features.\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "        if (typeof window.interactive_beam_jquery == 'undefined') {\n",
       "          var jqueryScript = document.createElement('script');\n",
       "          jqueryScript.src = 'https://code.jquery.com/jquery-3.4.1.slim.min.js';\n",
       "          jqueryScript.type = 'text/javascript';\n",
       "          jqueryScript.onload = function() {\n",
       "            var datatableScript = document.createElement('script');\n",
       "            datatableScript.src = 'https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js';\n",
       "            datatableScript.type = 'text/javascript';\n",
       "            datatableScript.onload = function() {\n",
       "              window.interactive_beam_jquery = jQuery.noConflict(true);\n",
       "              window.interactive_beam_jquery(document).ready(function($){\n",
       "                \n",
       "              });\n",
       "            }\n",
       "            document.head.appendChild(datatableScript);\n",
       "          };\n",
       "          document.head.appendChild(jqueryScript);\n",
       "        } else {\n",
       "          window.interactive_beam_jquery(document).ready(function($){\n",
       "            \n",
       "          });\n",
       "        }"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sayakpaul/.local/bin/.virtualenvs/tf/lib/python3.8/site-packages/apache_beam/dataframe/io.py:566: FutureWarning: WriteToFiles is experimental.\n",
      "  return pcoll | fileio.WriteToFiles(\n",
      "/Users/sayakpaul/.local/bin/.virtualenvs/tf/lib/python3.8/site-packages/apache_beam/io/fileio.py:535: BeamDeprecationWarning: options is deprecated since First stable release. References to <pipeline>.options will not be supported\n",
      "  p.options.view_as(GoogleCloudOptions).temp_location or\n",
      "warning: sdist: standard file not found: should have one of README, README.rst, README.txt, README.md\n",
      "\n",
      "warning: check: missing required meta-data: url\n",
      "\n",
      "warning: check: missing meta-data: either (author and author_email) or (maintainer and maintainer_email) must be supplied\n",
      "\n",
      "WARNING:root:Make sure that locally built Python SDK docker image has Python 3.8 interpreter.\n",
      "WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--gcs_location', 'gs://arxiv-data-nlp', 'True']\n",
      "WARNING:apache_beam.options.pipeline_options:Discarding unparseable args: ['--gcs_location', 'gs://arxiv-data-nlp', 'True']\n"
     ]
    }
   ],
   "source": [
    "with beam.Pipeline(argv=pipeline_args) as pipeline:\n",
    "    keywords = pipeline | beam.Create(query_keywords)\n",
    "    records = keywords | beam.FlatMap(query_with_keywords).with_output_types(ArxivEntries)\n",
    "    _ = to_dataframe(records).to_csv(\n",
    "        f\"gs://{gcs_bucket_name}/arxiv/sample.csv\", index=False\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cb97c245",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Updates are available for some Cloud SDK components.  To install them,\n",
      "please run:\n",
      "  $ gcloud components update\n",
      "\n",
      "gs://arxiv-data-nlp/arxiv/:\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00000-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00001-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00002-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00003-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00004-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00005-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00006-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00007-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00008-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00009-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00010-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00011-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00012-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00013-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00014-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00015-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00016-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00017-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00018-of-00020\n",
      "gs://arxiv-data-nlp/arxiv/sample.csv-00019-of-00020\n"
     ]
    }
   ],
   "source": [
    "!gsutil ls -R gs://{gcs_bucket_name}/arxiv/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f98d917f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying gs://arxiv-data-nlp/arxiv/sample.csv-00000-of-00020...\n",
      "| [1 files][  1.9 MiB/  1.9 MiB]  347.4 KiB/s                                   \n",
      "Operation completed over 1 objects/1.9 MiB.                                      \n"
     ]
    }
   ],
   "source": [
    "!gsutil cp gs://arxiv-data-nlp/arxiv/sample.csv-00000-of-00020 ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "daedd023",
   "metadata": {
    "id": "daedd023",
    "outputId": "e59c8622-02ca-4bfe-adfc-cec0208602fc"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>terms</th>\n",
       "      <th>titles</th>\n",
       "      <th>abstracts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>['cs.LG', 'cs.AI']</td>\n",
       "      <td>Self-supervised Learning on Graphs: Contrastiv...</td>\n",
       "      <td>Deep learning on graphs has recently achieved ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>['cs.CV', 'cs.CL']</td>\n",
       "      <td>Contrastive Video-Language Segmentation</td>\n",
       "      <td>We focus on the problem of segmenting a certai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>['cs.LG']</td>\n",
       "      <td>What to Prioritize? Natural Language Processin...</td>\n",
       "      <td>Managing large numbers of incoming bug reports...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>['cs.CV', 'cs.RO']</td>\n",
       "      <td>The VVAD-LRS3 Dataset for Visual Voice Activit...</td>\n",
       "      <td>Robots are becoming everyday devices, increasi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>['cs.CV']</td>\n",
       "      <td>UTNet: A Hybrid Transformer Architecture for M...</td>\n",
       "      <td>Transformer architecture has emerged to be suc...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                terms                                             titles  \\\n",
       "0  ['cs.LG', 'cs.AI']  Self-supervised Learning on Graphs: Contrastiv...   \n",
       "1  ['cs.CV', 'cs.CL']            Contrastive Video-Language Segmentation   \n",
       "2           ['cs.LG']  What to Prioritize? Natural Language Processin...   \n",
       "3  ['cs.CV', 'cs.RO']  The VVAD-LRS3 Dataset for Visual Voice Activit...   \n",
       "4           ['cs.CV']  UTNet: A Hybrid Transformer Architecture for M...   \n",
       "\n",
       "                                           abstracts  \n",
       "0  Deep learning on graphs has recently achieved ...  \n",
       "1  We focus on the problem of segmenting a certai...  \n",
       "2  Managing large numbers of incoming bug reports...  \n",
       "3  Robots are becoming everyday devices, increasi...  \n",
       "4  Transformer architecture has emerged to be suc...  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "\n",
    "df = pd.read_csv(\"sample.csv-00000-of-00020\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cba00e9",
   "metadata": {
    "id": "7cba00e9"
   },
   "source": [
    "## Acknowledgements\n",
    "\n",
    "* [Lukas Schwab](https://github.com/lukasschwab)\n",
    "* [Robert Bradshaw](https://www.linkedin.com/in/robert-bradshaw-1b48a07/)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "beam_arxiv.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}